JSON Data from api

Question

I would like to extract some JSON data from the following url: http://weather.news24.com/sa/johannesburg.

I used the Google developer tools which gave me some JSON and when I tried some online JSON validators, it's failing somehow on dates. Is there anyway I can get around the issue and correctly get the selected city JSON data?

Any help will be appreciated.

Answer 1

If the site does not offer a RESful service, then you may need to ask them if you want to scrape their page. Web page scraping is very tedious and the site's DOM structure could change at any time. There is no guarantee.

---

Page Scraping

You can scrape the page and convert the table data into a JSON object. I copied the DOM of the webpage as a bassis for this implementation.

If you generate the results below, the following response should be at the bottom.

{
  "windSpeed": "13km/h",
  "windDirection": "NW",
  "humidity": "35%",
  "dewPoint": "14",
  "comfortLevel": "30°C",
  "chanceOfRain": "10%",
  "sunrise": "5:57AM",
  "sunset": "6:45PM",
  "moonrise": "6:34PM",
  "moonset": "5:30AM",
  "moonPhase": "Full moon"
}

Example

// jQuery Plugins
(function($) {
    $.reduce = function(arr, fnReduce, valueInitial) {
        if (Array.prototype.reduce) {
            return Array.prototype.reduce.call(arr, fnReduce, valueInitial);
        }
        $.each(arr, function(index, value) {
            valueInitial = fnReduce.call(null, valueInitial, value, index, arr);
        });
        return valueInitial;
    };
})(jQuery);

// Used to convert a sequence of words to camelCase.
function camelCase(input) { 
    return input.toLowerCase().replace(/\s(.)/g, function(match, group1) {
        return group1.toUpperCase();
    });
}

// Main
$(function() {
  // Replace relative links with absolute.
  $('img').each(function() {
    $(this).attr('src', 'http://weather.news24.com/' + $(this).attr('src'));
  });

  // Find all table rows and grab the key-value pairs.
  var data = $.reduce($('#forecastContent tr'), function(map, tr) {
    var td = $(tr).find('td'), key, value;
    if (td.length === 2) {
      key = $(td.get(0)).text();
      value = $(td.get(1)).text();
    } else if (td.length === 3) {
      key = $(td.get(1)).text();
      value = $(td.get(2)).text();
    }
    if (key != null && key.trim().length > 0) {
     map[camelCase(key)] = value;
    }
    return map;
  }, {});

  $('#response').html(JSON.stringify(data, null, 2));
});

body {
  background: #FFF;
}
.bold {
  font-weight: bold;
}
.currentConditionsHeader {
  color: #1F38B7;
  font-weight: bold;
}
.forecast7DayBlockEmbedded {
  width: 80px;
  height: 70px;
  border: 1px solid #EAEFF2;
  text-align: center;
  padding-top: 10px;
  float: left;
  white-space: nowrap;
  margin-left: 13px;
  margin-top: 5px;
  margin-bottom: 15px;
  font-family: Verdana;
  font-size: 10px;
  font-weight: bold;
}
.forecastHeading {
  color: #000000;
  background-color: #D6F1FC;
  font-weight: bold;
  padding: 3px;
}
#response {
  background: #EEE;
  display: block;
}

<script src="https://ajax.googleapis.com/ajax/libs/jquery/1.11.1/jquery.min.js"></script>

<div id="forecastContent">
  <table cellpadding="0" celspacing="0" border="0">
    <tbody>
      <tr>
        <td>
          <div id="divCurrentOne" class="currentConditionsOneTable" style="white-space:nowrap;">
            <div class="float-l">
              <table cellpadding="2" cellspacing="2" border="0" class="currentConditionsOneTable">
                <tbody>
                  <tr>
                    <td colspan="2" class="currentConditionsHeader" style="vertical-align:middle;"> <img tyle="vertical-align:middle;" src="/Images/Icons/Forecasts/2.gif">LOW 17°C &nbsp;&nbsp; HIGH 24°C </td>
                  </tr>
                  <!-- Provide space and set column widths -->
                  <tr>
                    <td style="width:110px;">&nbsp;</td>
                    <td style="width:110px;">&nbsp;</td>
                  </tr>
                  <!-- end spacer and width definition row -->
                  <tr>
                    <td>Wind Speed</td>
                    <td class="bold">13km/h</td>
                  </tr>
                  <tr>
                    <td>Wind Direction</td>
                    <td class="bold">NW</td>
                  </tr>
                  <tr>
                    <td>Humidity</td>
                    <td class="bold">35%</td>
                  </tr>
                  <tr>
                    <td>Dew Point</td>
                    <td class="bold">14</td>
                  </tr>
                  <tr>
                    <td>Comfort Level</td>
                    <td class="bold">30°C</td>
                  </tr>
                  <tr>
                    <td>Chance of Rain</td>
                    <td class="bold">10%</td>
                  </tr>
                </tbody>
              </table>
            </div>
          </div>
        </td>
        <td>
          <div id="divAstronomy" class="currentConditionsAstronomyTable" style="white-space:nowrap;">
            <div class="float-l">
              <table cellpadding="2" cellspacing="2" border="0" class="currentConditionsAstronomyTable">
                <tbody>
                  <tr>
                    <td colspan="3">&nbsp;</td>
                  </tr>
                  <!-- Provide space and set column widths -->
                  <tr>
                    <td>&nbsp;</td>
                    <td>&nbsp;</td>
                    <td style="width:100%;">&nbsp;</td>
                  </tr>
                  <!-- end spacer and width definition row -->
                  <tr>
                    <td>&nbsp;</td>
                    <td>Sunrise</td>
                    <td class="bold">5:57AM</td>
                  </tr>
                  <tr>
                    <td>&nbsp;</td>
                    <td>Sunset</td>
                    <td class="bold">6:45PM</td>
                  </tr>
                  <tr>
                    <td>&nbsp;</td>
                    <td>&nbsp;</td>
                    <td class="bold">&nbsp;</td>
                  </tr>
                  <tr>
                    <td rowspan="2" class="currentConditionsMoonImage"><img src="/Images/Icons/Moon/cw_full_moon.gif"></td>
                    <td>Moonrise</td>
                    <td class="bold">6:34PM</td>
                  </tr>
                  <tr>
                    <td>Moonset</td>
                    <td class="bold">5:30AM</td>
                  </tr>
                  <tr>
                    <td>&nbsp;</td>
                    <td>Moon Phase</td>
                    <td class="bold">Full moon</td>
                  </tr>
                </tbody>
              </table>
            </div>
          </div>
        </td>
      </tr>
      <tr>
        <td colspan="2" id="div7DayForecast" style="width:100%;white-space:nowrap;">
          <div class="forecast7DayEmbeddedContainer">
            <div class="forecast7DayContainer">
              <br>
              <br>
              <div id="7dayforecast" class="forecastHeading">7 Day Forecast</div>
              <div class="forecast7DayBlockEmbedded"> Tuesday
                <br> <img src="/Images/Icons/Forecasts/2.gif">
                <br> 17°C - 27°C </div>
              <div class="forecast7DayBlockEmbedded"> Wednesday
                <br> <img src="/Images/Icons/Forecasts/33.gif">
                <br> 17°C - 28°C </div>
              <div class="forecast7DayBlockEmbedded"> Thursday
                <br> <img src="/Images/Icons/Forecasts/33.gif">
                <br> 16°C - 29°C </div>
              <div class="forecast7DayBlockEmbedded"> Friday
                <br> <img src="/Images/Icons/Forecasts/33.gif">
                <br> 17°C - 28°C </div>
              <div class="forecast7DayBlockEmbedded"> Saturday
                <br> <img src="/Images/Icons/Forecasts/4.gif">
                <br> 16°C - 27°C </div>
              <div class="forecast7DayBlockEmbedded"> Sunday
                <br> <img src="/Images/Icons/Forecasts/22.gif">
                <br> 16°C - 26°C </div>
              <div class="forecast7DayBlockEmbedded"> Monday
                <br> <img src="/Images/Icons/Forecasts/4.gif">
                <br> 15°C - 27°C </div>
            </div>
          </div>
        </td>
      </tr>
    </tbody>
  </table>
</div>
<hr />
<h2>Response</h2>
<pre>
  <code id="response"></code>
</pre>

---

RESTful Options

There are free REST APIs that return JSON weather data. All you need to do is sign up and receive a key.

OpenWeatherMap

API: http://openweathermap.org/api

Request: http://api.openweathermap.org/data/2.5/weather?q=johannesburg,za&appid=YOUR_KEY

Response

{
    coord: {
        lon: 28.04,
        lat: -26.2
    },
    weather: [{
        id: 801,
        main: "Clouds",
        description: "few clouds",
        icon: "02d"
    }],
    base: "cmc stations",
    main: {
        temp: 304.367,
        pressure: 852.81,
        humidity: 26,
        temp_min: 304.367,
        temp_max: 304.367,
        sea_level: 1022.08,
        grnd_level: 852.81
    },
    wind: {
        speed: 2.51,
        deg: 296.507
    },
    clouds: {
        all: 20
    },
    dt: 1456147770,
    sys: {
        message: 0.0116,
        country: "ZA",
        sunrise: 1456113434,
        sunset: 1456159501
    },
    id: 993800,
    name: "Johannesburg",
    cod: 200
}

Weather Underground

API: https://www.wunderground.com/weather/api/

Request: http://api.wunderground.com/api/YOUR_KEY/conditions/q/ZA/Johannesburg.json

Response

{
    response: {
        version: "0.1",
        termsofService: "http://www.wunderground.com/weather/api/d/terms.html",
        features: {
            conditions: 1
        }
    },
    current_observation: {
        image: {
            url: "http://icons.wxug.com/graphics/wu2/logo_130x80.png",
            title: "Weather Underground",
            link: "http://www.wunderground.com"
        },
        display_location: {
            full: "Johannesburg, South Africa",
            city: "Johannesburg",
            state: "",
            state_name: "South Africa",
            country: "ZA",
            country_iso3166: "ZA",
            zip: "00000",
            magic: "1",
            wmo: "68368",
            latitude: "-26.12999916",
            longitude: "28.22999954",
            elevation: "1700.00000000"
        },
        observation_location: {
            full: "Johannesburg, ",
            city: "Johannesburg",
            state: "",
            country: "ZA",
            country_iso3166: "ZA",
            latitude: "-26.13077927",
            longitude: "28.24378967",
            elevation: "5643 ft"
        },
        estimated: {},
        station_id: "FAOR",
        observation_time: "Last Updated on February 22, 3:30 PM SAST",
        observation_time_rfc822: "Mon, 22 Feb 2016 15:30:00 +0200",
        observation_epoch: "1456147800",
        local_time_rfc822: "Mon, 22 Feb 2016 15:44:41 +0200",
        local_epoch: "1456148681",
        local_tz_short: "SAST",
        local_tz_long: "Africa/Johannesburg",
        local_tz_offset: "+0200",
        weather: "Scattered Clouds",
        temperature_string: "84 F (29 C)",
        temp_f: 84,
        temp_c: 29,
        relative_humidity: "29%",
        wind_string: "From the WNW at 10 MPH",
        wind_dir: "WNW",
        wind_degrees: 290,
        wind_mph: 10,
        wind_gust_mph: 0,
        wind_kph: 17,
        wind_gust_kph: 0,
        pressure_mb: "1020",
        pressure_in: "30.12",
        pressure_trend: "0",
        dewpoint_string: "48 F (9 C)",
        dewpoint_f: 48,
        dewpoint_c: 9,
        heat_index_string: "NA",
        heat_index_f: "NA",
        heat_index_c: "NA",
        windchill_string: "NA",
        windchill_f: "NA",
        windchill_c: "NA",
        feelslike_string: "84 F (29 C)",
        feelslike_f: "84",
        feelslike_c: "29",
        visibility_mi: "6.2",
        visibility_km: "10.0",
        solarradiation: "--",
        UV: "6",
        precip_1hr_string: "-9999.00 in (-9999.00 mm)",
        precip_1hr_in: "-9999.00",
        precip_1hr_metric: "--",
        precip_today_string: "0.00 in (0.0 mm)",
        precip_today_in: "0.00",
        precip_today_metric: "0.0",
        icon: "partlycloudy",
        icon_url: "http://icons.wxug.com/i/c/k/partlycloudy.gif",
        forecast_url: "http://www.wunderground.com/global/stations/68368.html",
        history_url: "http://www.wunderground.com/history/airport/FAOR/2016/2/22/DailyHistory.html",
        ob_url: "http://www.wunderground.com/cgi-bin/findweather/getForecast?query=-26.13077927,28.24378967",
        nowcast: ""
    }
}