Data processing started with identifying the data structure setup by Node-RED. Node-RED being built on Node.js means the default method of data interaction will be with JSON. Luckily, there is a native JSON interpreter for Python which make data loading easy!
Parsing the file is quick and easy, and loads the data file as a Python dictionary. We have an error counter, but over the 111,500 records process, only 3 errors were encountered.
The a[i] value above is the numpy array used to process the data, we stored that after loading for ease of use in the future.
{
"ID": "ObserverIP",
"PASSWORD": "E8:68:E7:11:57:67",
"tempf": "45.50",
"humidity": "79",
"dewptf": "39.38",
"windchillf": "45.50",
"winddir": "284",
"windspeedmph": "2.46",
"windgustmph": "5.82",
"rainin": "0.000",
"dailyrainin": "0.000",
"weeklyrainin": "0.642",
"monthlyrainin": "0.642",
"yearlyrainin": "6.752",
"solarradiation": "105.33",
"UV": "1",
"indoortempf": "64.58",
"indoorhumidity": "43",
"baromin": "29.554",
"temp2f": "69.62",
"humidity2": "35",
"temp3f": "64.76",
"humidity3": "42",
"temp4f": "50.54",
"humidity4": "78",
"temp5f": "47.48",
"humidity5": "65",
"temp6f": "4.64",
"temp7f": "62.96",
"humidity7": "44",
"soilmoisture": "13",
"lowbatt": "0",
"dateutc": "now",
"softwaretype": "OBSERVERIP2_V2.2.6",
"action": "updateraw",
"realtime": "1",
"rtfreq": "5",
"ts": 1712260423184,
"lastUpdate": "2024-04-04T19:53:43.184Z",
"lastUpdateStr": "2024-04-04T15:53:43",
"latitude": 42.561195,
"longitude": -83.638824,
"angleType": "deg",
"azimuth": 229.0998005013489,
"altitude": 42.665583204360544,
"altitudeDegrees": 42.665583204360544,
"azimuthDegrees": 229.0998005013489,
"altitudeRadians": 0.7446549041996842,
"azimuthRadians": 3.9985458344106943,
"times": {
"solarNoon": {
"value": "2024-04-04T17:38:30.392Z",
"ts": 1712252310392.7122,
"name": "solarNoon",
"julian": 2460405.2350739897,
"valid": true,
"pos": 10
},
"nadir": {
"value": "2024-04-05T05:38:30.392Z",
"ts": 1712295510392.7122,
"name": "nadir",
"julian": 2460405.7350739897,
"valid": true,
"pos": 21
},
"goldenHourDuskStart": {
"value": "2024-04-04T23:27:37.701Z",
"ts": 1712273257701.4224,
"name": "goldenHourDuskStart",
"elevation": 6,
"julian": 2460405.4775196924,
"valid": true,
"pos": 11
},
"goldenHourDawnEnd": {
"value": "2024-04-04T11:49:23.084Z",
"ts": 1712231363084.002,
"name": "goldenHourDawnEnd",
"elevation": 6,
"julian": 2460404.992628287,
"valid": true,
"pos": 9
},
"sunsetStart": {
"value": "2024-04-05T00:02:00.633Z",
"ts": 1712275320633.6648,
"name": "sunsetStart",
"elevation": -0.3,
"julian": 2460405.501396223,
"valid": true,
"pos": 12
},
"sunriseEnd": {
"value": "2024-04-04T11:15:00.151Z",
"ts": 1712229300151.7593,
"name": "sunriseEnd",
"elevation": -0.3,
"julian": 2460404.9687517565,
"valid": true,
"pos": 8
},
"sunsetEnd": {
"value": "2024-04-05T00:04:56.271Z",
"ts": 1712275496271.5774,
"name": "sunsetEnd",
"elevation": -0.833,
"julian": 2460405.503429069,
"valid": true,
"pos": 13
},
"sunriseStart": {
"value": "2024-04-04T11:12:04.513Z",
"ts": 1712229124513.8467,
"name": "sunriseStart",
"elevation": -0.833,
"julian": 2460404.9667189103,
"valid": true,
"pos": 7
},
"goldenHourDuskEnd": {
"value": "2024-04-05T00:05:51.350Z",
"ts": 1712275551350.5383,
"name": "goldenHourDuskEnd",
"elevation": -1,
"julian": 2460405.504066557,
"valid": true,
"pos": 14
},
"goldenHourDawnStart": {
"value": "2024-04-04T11:11:09.434Z",
"ts": 1712229069434.8857,
"name": "goldenHourDawnStart",
"elevation": -1,
"julian": 2460404.9660814223,
"valid": true,
"pos": 6
},
"blueHourDuskStart": {
"value": "2024-04-05T00:22:25.270Z",
"ts": 1712276545270.5547,
"name": "blueHourDuskStart",
"elevation": -4,
"julian": 2460405.515570261,
"valid": true,
"pos": 15
},
"blueHourDawnEnd": {
"value": "2024-04-04T10:54:35.514Z",
"ts": 1712228075514.8694,
"name": "blueHourDawnEnd",
"elevation": -4,
"julian": 2460404.9545777184,
"valid": true,
"pos": 5
},
"civilDusk": {
"value": "2024-04-05T00:33:33.500Z",
"ts": 1712277213500.6458,
"name": "civilDusk",
"elevation": -6,
"julian": 2460405.5233044056,
"valid": true,
"pos": 16
},
"civilDawn": {
"value": "2024-04-04T10:43:27.284Z",
"ts": 1712227407284.7783,
"name": "civilDawn",
"elevation": -6,
"julian": 2460404.946843574,
"valid": true,
"pos": 4
},
"blueHourDuskEnd": {
"value": "2024-04-05T00:44:47.295Z",
"ts": 1712277887295.1404,
"name": "blueHourDuskEnd",
"elevation": -8,
"julian": 2460405.531102953,
"valid": true,
"pos": 17
},
"blueHourDawnStart": {
"value": "2024-04-04T10:32:13.490Z",
"ts": 1712226733490.2837,
"name": "blueHourDawnStart",
"elevation": -8,
"julian": 2460404.9390450264,
"valid": true,
"pos": 3
},
"nauticalDusk": {
"value": "2024-04-05T01:07:35.889Z",
"ts": 1712279255889.5276,
"name": "nauticalDusk",
"elevation": -12,
"julian": 2460405.546943166,
"valid": true,
"pos": 18
},
"nauticalDawn": {
"value": "2024-04-04T10:09:24.895Z",
"ts": 1712225364895.8967,
"name": "nauticalDawn",
"elevation": -12,
"julian": 2460404.9232048136,
"valid": true,
"pos": 2
},
"amateurDusk": {
"value": "2024-04-05T01:25:05.860Z",
"ts": 1712280305860.9797,
"name": "amateurDusk",
"elevation": -15,
"julian": 2460405.559095613,
"valid": true,
"pos": 19
},
"amateurDawn": {
"value": "2024-04-04T09:51:54.924Z",
"ts": 1712224314924.4443,
"name": "amateurDawn",
"elevation": -15,
"julian": 2460404.9110523663,
"valid": true,
"pos": 1
},
"astronomicalDusk": {
"value": "2024-04-05T01:43:02.248Z",
"ts": 1712281382248.825,
"name": "astronomicalDusk",
"elevation": -18,
"julian": 2460405.571553806,
"valid": true,
"pos": 20
},
"astronomicalDawn": {
"value": "2024-04-04T09:33:58.536Z",
"ts": 1712223238536.5994,
"name": "astronomicalDawn",
"elevation": -18,
"julian": 2460404.8985941736,
"valid": true,
"pos": 0
},
"dawn": {
"value": "2024-04-04T10:43:27.284Z",
"ts": 1712227407284.7783,
"name": "civilDawn",
"elevation": -6,
"julian": 2460404.946843574,
"valid": true,
"pos": -2,
"deprecated": true,
"posOrg": 4
},
"dusk": {
"value": "2024-04-05T00:33:33.500Z",
"ts": 1712277213500.6458,
"name": "civilDusk",
"elevation": -6,
"julian": 2460405.5233044056,
"valid": true,
"pos": -2,
"deprecated": true,
"posOrg": 16
},
"nightEnd": {
"value": "2024-04-04T09:33:58.536Z",
"ts": 1712223238536.5994,
"name": "astronomicalDawn",
"elevation": -18,
"julian": 2460404.8985941736,
"valid": true,
"pos": -2,
"deprecated": true,
"posOrg": 0
},
"night": {
"value": "2024-04-05T01:43:02.248Z",
"ts": 1712281382248.825,
"name": "astronomicalDusk",
"elevation": -18,
"julian": 2460405.571553806,
"valid": true,
"pos": -2,
"deprecated": true,
"posOrg": 20
},
"nightStart": {
"value": "2024-04-05T01:43:02.248Z",
"ts": 1712281382248.825,
"name": "astronomicalDusk",
"elevation": -18,
"julian": 2460405.571553806,
"valid": true,
"pos": -2,
"deprecated": true,
"posOrg": 20
},
"goldenHour": {
"value": "2024-04-04T23:27:37.701Z",
"ts": 1712273257701.4224,
"name": "goldenHourDuskStart",
"elevation": 6,
"julian": 2460405.4775196924,
"valid": true,
"pos": -2,
"deprecated": true,
"posOrg": 11
},
"sunrise": {
"value": "2024-04-04T11:12:04.513Z",
"ts": 1712229124513.8467,
"name": "sunriseStart",
"elevation": -0.833,
"julian": 2460404.9667189103,
"valid": true,
"pos": -2,
"deprecated": true,
"posOrg": 7
},
"sunset": {
"value": "2024-04-05T00:04:56.271Z",
"ts": 1712275496271.5774,
"name": "sunsetEnd",
"elevation": -0.833,
"julian": 2460405.503429069,
"valid": true,
"pos": -2,
"deprecated": true,
"posOrg": 13
},
"goldenHourEnd": {
"value": "2024-04-04T11:49:23.084Z",
"ts": 1712231363084.002,
"name": "goldenHourDawnEnd",
"elevation": 6,
"julian": 2460404.992628287,
"valid": true,
"pos": -2,
"deprecated": true,
"posOrg": 9
},
"goldenHourStart": {
"value": "2024-04-04T23:27:37.701Z",
"ts": 1712273257701.4224,
"name": "goldenHourDuskStart",
"elevation": 6,
"julian": 2460405.4775196924,
"valid": true,
"pos": -2,
"deprecated": true,
"posOrg": 11
}
},
"positionAtSolarNoon": {
"azimuth": 3.1463499949757145,
"altitude": 0.931302963586873,
"zenith": 0.6394933632080235,
"azimuthDegrees": 180.2725755831162,
"altitudeDegrees": 53.35972926155361,
"zenithDegrees": 36.64027073844639,
"declination": 0.10334462401411768
},
"altitudePercent": 79.95839520704176,
"pos": [],
"posChanged": false
}
Leave a Reply