Internal Server Error: GET https://link.to.website/ (Internal Server Error)

Question

I am attempting to create a personal use application that scrapes long and short data from a specific website. After scraping the data, the application should store it in a database and display it on a JavaScript chart. The chart should update every hour (for testing purposes, it's currently set to update every minute) and display the last 24 hours of scraped information. Everything works properly on my computer locally, even after creating a database with Render and connecting it. However, when I deploy the application on Render, I encounter numerous errors for which I don't know the cause. Here are the errors:

(index):27 Initializing chart with last 24 hours data: {error: '(psycopg2.OperationalError) could not translate ho… on this error at: https://sqlalche.me/e/20/e3q8)'} (index):36 Error initializing chart: TypeError: data.forEach is not a function at (index):28:18 (anonymous) @ (index):36 Promise.catch (async) initializeChart @ (index):36 (anonymous) @ (index):40 (index):20

GET https://link.to.website/get_last_24_hours_data_eurusd 500 (Internal Server Error) initializeChart @ (index):20 (anonymous) @ (index):136 (index):27 Initializing chart with last 24 hours data: {error: '(psycopg2.OperationalError) could not translate ho… on this error at: https://sqlalche.me/e/20/e3q8)'} (index):36 Error initializing chart: TypeError: data.forEach is not a function at (index):28:18 (anonymous) @ (index):36 Promise.catch (async) initializeChart @ (index):36 (anonymous) @ (index):136 (index):43

   GET https://link.to.website/get_last_24_hours_data_eurusd 500 (Internal Server Error)

fetchDataAndUpdateChart @ (index):43 (anonymous) @ (index):140 setInterval (async) (anonymous) @ (index):139 (index):50 Received data: {error: '(psycopg2.OperationalError) could not translate ho… on this error at: https://sqlalche.me/e/20/e3q8)'} (index):59 Error fetching data: TypeError: data.forEach is not a function at (index):51:18

It's important to mention that the route get_last_24_hours_data_eurusd is functioning perfectly locally, providing JSON-formatted data for the last 24 hours of updates. Additionally, the Render database successfully receives data when running locally, and everything operates smoothly. Any ideas? Here is the code:

Main.py

from flask import Flask, render_template, jsonify,request
from flask_sqlalchemy import SQLAlchemy
from bs4 import BeautifulSoup
import requests
from datetime import datetime, timedelta
from flask_socketio import SocketIO
import threading
import time
from threading import Lock
import os


app = Flask(__name__)
app.config['SQLALCHEMY_DATABASE_URI'] = os.environ.get("SQLALCHEMY_DATABASE_URI")
db = SQLAlchemy(app)
socketio = SocketIO(app)

class ForexData(db.Model):
    id = db.Column(db.Integer, primary_key=True)
    currency_pair = db.Column(db.String(10))
    timestamp = db.Column(db.DateTime, default=datetime.now)
    short = db.Column(db.Integer)
    long = db.Column(db.Integer)


data_lock = Lock()

def scrape_and_store_data(currency_pair, url):
    response = requests.get(url)

    if response.status_code == 200:
        soup = BeautifulSoup(response.text, 'html.parser')
        elements_with_class = soup.find_all("td")

        if len(elements_with_class) > 26:
            target_element_text_22 = elements_with_class[22].text
            target_element_text_26 = elements_with_class[26].text if elements_with_class[26] else None

            target_value_22 = int(target_element_text_22.replace('%', '').strip())

            if target_element_text_26 is not None:
                target_value_26 = int(target_element_text_26.replace('%', '').strip())

                with data_lock:
                    current_time = datetime.now().replace(second=0, microsecond=0)
                    existing_entry = ForexData.query.filter_by(currency_pair=currency_pair, timestamp=current_time).first()

                    if existing_entry:
                        print("Entry for current minute already exists. Skipping...")
                    else:
                       
                        data_entry = ForexData(currency_pair=currency_pair, short=target_value_22, long=target_value_26, timestamp=current_time)
                        db.session.add(data_entry)
                        db.session.commit()
                        print("Data stored for the current minute.")

                return {'short': target_value_22, 'long': target_value_26}

    return {'error': 'Failed to fetch and process data from the website'}



def start_scheduler():
    with app.app_context():
        while True:

            current_time = datetime.now()
            next_minute = current_time.replace(second=0, microsecond=0) + timedelta(minutes=1)
            wait_seconds = (next_minute - current_time).total_seconds()


            time.sleep(wait_seconds)

            scrape_and_store_data('EURUSD', 'https://www.myfxbook.com/community/outlook/EURUSD')


@app.route('/')
def index():
    return render_template('index.html')

@socketio.on('connect')
def handle_connect():
    print('Client connected')
    send_initial_data()

def send_initial_data():
    data = get_last_24_hours_data_eurusd()
    socketio.emit('initial_data', data)

@app.route('/get_last_24_hours_data_eurusd')
def get_last_24_hours_data_eurusd():
    try:
        with app.app_context():
            data = ForexData.query.filter_by(currency_pair='EURUSD').order_by(ForexData.timestamp.desc()).limit(24).all()
            formatted_data = [
                {'timestamp': entry.timestamp.isoformat(), 'short': entry.short, 'long': entry.long}
                for entry in reversed(data)
            ]
            return jsonify(formatted_data)
    except Exception as e:
        error_message = str(e)
        return jsonify({'error': error_message}), 500


if __name__ == '__main__':
    with app.app_context():
        db.create_all()
    threading.Thread(target=start_scheduler).start()
    socketio.run(app, debug=True)

Index.html

<!DOCTYPE html>
<html lang="en">
  <head>
    <meta charset="UTF-8" />
    <title>Real-time Graph EURUSD</title>
    <script src="https://cdnjs.cloudflare.com/ajax/libs/socket.io/4.3.2/socket.io.js"></script>
    <script src="https://cdn.jsdelivr.net/npm/chart.js"></script>
    <script src="https://cdn.jsdelivr.net/npm/chartjs-adapter-date-fns/dist/chartjs-adapter-date-fns.bundle.min.js"></script>
  </head>
  <body>
    <canvas id="myChart" width="800" height="400"></canvas>

    <script>
      function initializeChart() {
        console.log("Initializing chart...");

        fetch("/get_last_24_hours_data_eurusd", {
          headers: {
            Accept: "application/json",
          },
        })
          .then((response) => response.json())
          .then((data) => {
            console.log("Initializing chart with last 24 hours data:", data);
            data.forEach((entry) => {
              entry.timestamp = new Date(entry.timestamp);
            });
            myChart.data.labels = data.map((entry) => entry.timestamp);
            myChart.data.datasets[0].data = data.map((entry) => entry.short);
            myChart.data.datasets[1].data = data.map((entry) => entry.long);
            myChart.update();
          })
          .catch((error) => console.error("Error initializing chart:", error));
      }

      initializeChart();

      function fetchDataAndUpdateChart() {
        fetch("/get_last_24_hours_data_eurusd", {
          headers: {
            Accept: "application/json",
          },
        })
          .then((response) => response.json())
          .then((data) => {
            console.log("Received data:", data);
            data.forEach((entry) => {
              entry.timestamp = new Date(entry.timestamp);
            });
            myChart.data.labels = data.map((entry) => entry.timestamp);
            myChart.data.datasets[0].data = data.map((entry) => entry.short);
            myChart.data.datasets[1].data = data.map((entry) => entry.long);
            myChart.update();
          })
          .catch((error) => console.error("Error fetching data:", error));
      }

      function updateChart(data) {
        console.log("Chart Data (Before Update):", myChart.data);

        var currentTime = new Date();

        myChart.data.labels.push(currentTime);
        myChart.data.datasets[0].data.push(data.short);
        myChart.data.datasets[1].data.push(data.long);

        if (myChart.data.labels.length > 24) {
          myChart.data.labels.shift();
          myChart.data.datasets[0].data.shift();
          myChart.data.datasets[1].data.shift();
        }

        myChart.update();

        console.log("Chart Data (After Update):", myChart.data);
      }

      var ctx = document.getElementById("myChart").getContext("2d");

      var myChart = new Chart(ctx, {
        type: "line",
        data: {
          labels: [],
          datasets: [
            {
              label: "Short Line",
              data: [],
              borderColor: "red",
              borderWidth: 2,
              fill: false,
            },
            {
              label: "Long Line",
              data: [],
              borderColor: "green",
              borderWidth: 2,
              fill: false,
            },
          ],
        },
        options: {
          responsive: true,
          maintainAspectRatio: false,
          scales: {
            x: {
              type: "time",
              position: "bottom",
              time: {
                unit: "minute",
                stepSize: 1,
                displayFormats: {
                  minute: "HH:mm",
                },
              },
            },
            y: {
              max: 100,
              min: 0,
              stepSize: 10,
            },
          },
        },
      });

      initializeChart();

      setInterval(() => {
        fetchDataAndUpdateChart();
      }, 60 * 1000);
    </script>
  </body>
</html>