From 3c0b05d0b01958d38d883ac1a4402c77fb15bff8 Mon Sep 17 00:00:00 2001 From: JJ Brosnan <84038776+jjbrosnan@users.noreply.github.com> Date: Mon, 6 Dec 2021 11:33:32 -0500 Subject: [PATCH 1/6] Update examples so they work --- CryptoCurrency/script.py | 12 ++-- Fit/accessFit.py | 71 ++++++++++--------- Prometheus/app.d/prometheus.py | 2 +- TickingHeartRate/runTickingHeartRateReplay.py | 35 ++++----- 4 files changed, 63 insertions(+), 57 deletions(-) diff --git a/CryptoCurrency/script.py b/CryptoCurrency/script.py index a9a3eef0..dda3764d 100644 --- a/CryptoCurrency/script.py +++ b/CryptoCurrency/script.py @@ -1,9 +1,11 @@ import os -os.system("pip install pycoingecko") +try: + from pycoingecko import CoinGeckoAPI +except ImportError: + os.system("pip install pycoingecko") + from pycoingecko import CoinGeckoAPI -from pycoingecko import CoinGeckoAPI - -from deephaven.DBTimeUtils import secondsToTime, millisToTime +from deephaven.DateTimeUtils import secondsToTime, millisToTime from deephaven.TableTools import merge from deephaven import DynamicTableWriter import deephaven.Types as dht @@ -63,4 +65,4 @@ def thread_func(): sleep(secondsToSleep) thread = threading.Thread(target = thread_func) -thread.start() +thread.start() \ No newline at end of file diff --git a/Fit/accessFit.py b/Fit/accessFit.py index eb93705b..0169e0c8 100644 --- a/Fit/accessFit.py +++ b/Fit/accessFit.py @@ -1,36 +1,39 @@ from deephaven import DynamicTableWriter, Types as dht -from deephaven.DBTimeUtils import convertDateTime +from deephaven.DateTimeUtils import convertDateTime # Ensure fitparse is installed. import os -os.system("pip install fitparse") -from fitparse import FitFile +try: + from fitparse import FitFile +except ImportError: + os.system("pip install fitparse") + from fitparse import FitFile # Change to the name of the downloaded file (including any intermediate directory added to docker) -fitfile = FitFile('/data/ThursMorn.fit') +fitfile = FitFile('/data/examples/Fit/ThursMorn.fit') # See the size of your file records = list(fitfile.get_messages('record')) -print("Number of data points:" + str(len(records))) +print("Number of data points: {}".format(len(records))) # Setup deephaven tables to hold results # Heart rate -columnNames = ["Timestamp", "Heart Rate"] -columnTypes = [dht.datetime, dht.int_] -hrTableWriter = DynamicTableWriter(columnNames, columnTypes) -heartRateData = hrTableWriter.getTable() +column_names = ["Timestamp", "Heart Rate"] +column_types = [dht.datetime, dht.int_] +hr_table_writer = DynamicTableWriter(column_names, column_types) +heart_rate_data = hrTableWriter.getTable() # Gps data -columnNames = ["Timestamp", "Enhanced Altitude m", "Enhanced Speed m/s", "GPPS Accuracy m", "Position lat semicircles", "Position long semicircles", "Speed m/s"] -columnTypes = [dht.datetime, dht.double, dht.double, dht.int_, dht.int_, dht.int_, dht.double] -gpsTableWriter = DynamicTableWriter(columnNames, columnTypes) -gpsData = gpsTableWriter.getTable() +column_names = ["Timestamp", "Enhanced Altitude m", "Enhanced Speed m/s", "GPPS Accuracy m", "Position lat semicircles", "Position long semicircles", "Speed m/s"] +column_types = [dht.datetime, dht.double, dht.double, dht.int_, dht.int_, dht.int_, dht.double] +gps_table_writer = DynamicTableWriter(column_names, column_types) +gps_data = gps_table_writer.getTable() # Set timezone based on preferences. Fit data may not include timezone -timezone="MT" -timezone=" " + timezone # Ensure there is a blank space before timezone for later parsing. +timezone = "MT" +timezone = " " + timezone # Ensure there is a blank space before timezone for later parsing. # Process in smaller batches first, until you are happy with the results you are working with counter = 20 -counter = len(records)+1 +counter = len(records) + 1 ## Debug your files by looking at individual records at a time. # record = records[0] @@ -60,25 +63,25 @@ # Go through all the data entries in this record items=list(record) - if (mode == "hr" ): - rawHeartRate=str(items[0]).split(" ")[1] - finalHeartRate=int(rawHeartRate) - rawTime=str(items[1]).split(" ")[1] - finalTime=convertDateTime(rawTime.replace(" ", "T") + timezone) - hrTableWriter.logRow(finalTime, finalHeartRate) + if (mode == "hr"): + raw_heart_rate = str(items[0]).split()[1] + final_heart_rate = int(raw_heart_rate) + raw_time = str(items[1]).split()[1] + final_time = convertDateTime(raw_time.replace(" ", "T") + timezone) + hr_table_writer.logRow(final_time, final_heart_rate) - if (mode == "gps" ): - rawTime=str(items[6])[11:30] - finalTime=convertDateTime(rawTime.replace(" ", "T") + timezone) + if (mode == "gps"): + raw_time = str(items[6])[11:30] + final_time = convertDateTime(raw_time.replace(" ", "T") + timezone) - finalAltitude=float(str(items[0]).split(" ")[1]) - finalEnhSpeed=float(str(items[1]).split(" ")[1]) - finalPosLat=int(str(items[3]).split(" ")[1]) - finalPosLong=int(str(items[4]).split(" ")[1]) - finalSpeed=float(str(items[5]).split(" ")[1]) + final_altitude = float(str(items[0]).split()[1]) + final_enh_speed = float(str(items[1]).split()[1]) + final_pos_lat = int(str(items[3]).split()[1]) + final_pos_long = int(str(items[4]).split()[1]) + final_speed = float(str(items[5]).split()[1]) - rawGPSAcc=str(items[2]).split(" ")[1] + raw_gps_acc = str(items[2]).split()[1] # If preferred, the col type for GPS could be set as String, then further processing done even when value is None - if rawGPSAcc != "None": - finalGPSAcc=int(str(items[2]).split(" ")[1]) - gpsTableWriter.logRow(finalTime, finalAltitude,finalEnhSpeed,finalGPSAcc,finalPosLat,finalPosLong,finalSpeed) + if raw_gps_acc != "None": + final_gps_acc = int(str(items[2]).split()[1]) + gps_table_writer.logRow(final_time, final_altitude, final_enh_speed, final_gps_acc, final_pos_lat, final_pos_long, final_speed) \ No newline at end of file diff --git a/Prometheus/app.d/prometheus.py b/Prometheus/app.d/prometheus.py index 3ceaa5dd..016a99d3 100644 --- a/Prometheus/app.d/prometheus.py +++ b/Prometheus/app.d/prometheus.py @@ -14,7 +14,7 @@ """ from deephaven.TableTools import newTable, stringCol, dateTimeCol, doubleCol from deephaven import DynamicTableWriter -from deephaven.DBTimeUtils import millisToTime +from deephaven.DateTimeUtils import millisToTime import deephaven.Types as dht from typing import Callable diff --git a/TickingHeartRate/runTickingHeartRateReplay.py b/TickingHeartRate/runTickingHeartRateReplay.py index dc793b7a..e452ee16 100644 --- a/TickingHeartRate/runTickingHeartRateReplay.py +++ b/TickingHeartRate/runTickingHeartRateReplay.py @@ -1,34 +1,35 @@ -from deephaven.TableTools import readHeaderlessCsv +from deephaven import csv from deephaven import DynamicTableWriter, Types as dht -from deephaven import DBTimeUtils -from deephaven.DBTimeUtils import autoEpochToTime +from deephaven import DateTimeUtils import pathlib import time import threading +import sys + # Max number of csv files to pull in -csvFiles=500 +csv_files=500 # Setup deephaven tables to hold Heart Rate results -columnNames = ["Timestamp", "Heart Rate"] -columnTypes = [dht.datetime, dht.int_] -hrTableWriter = DynamicTableWriter(columnNames, columnTypes) -heartRateData = hrTableWriter.getTable() +column_names = ["Timestamp", "Heart Rate"] +column_types = [dht.datetime, dht.int_] +hr_table_writer = DynamicTableWriter(column_names, column_types) +heart_rate_data = hr_table_writer.getTable() # Function to log data to the dynamic table def thread_func(): - for x in range(1,csvFiles): - nextFile=("/data/csv/%d.csv" % x) - print(nextFile) - path = pathlib.Path(nextFile) + for x in range(1, csv_files): + next_file = ("/data/examples/TickingHeartRate/csv/%d.csv" % x) + print(next_file) + path = pathlib.Path(next_file) if path.exists() and path.is_file(): - nextHR=readHeaderlessCsv(nextFile).update("Timestamp=Column1", "Heart_rate=Column2").select("Timestamp", "Heart_rate") - nextRecord=nextHR.getRecord(0, "Timestamp", "Heart_rate") - timestamp=DBTimeUtils.autoEpochToTime(nextRecord[0]) - hrTableWriter.logRow(timestamp, int(nextRecord[1])) + next_hr = csv.read(next_file, headless = True).update("Timestamp=Column1", "Heart_rate=Column2").select("Timestamp", "Heart_rate") + next_record = next_hr.getRecord(0, "Timestamp", "Heart_rate") + timestamp = next_record[0] + hr_table_writer.logRow(timestamp, int(next_record[1])) time.sleep(1) else: - print("File does not exist: " + nextFile) + print("File does not exist: " + next_file) # Thread to log data to the dynamic table thread = threading.Thread(target = thread_func) From 711b2be9033a8bae541d778bbb9ae172bf9a474a Mon Sep 17 00:00:00 2001 From: Amanda L Martin Date: Tue, 7 Dec 2021 16:29:57 -0500 Subject: [PATCH 2/6] Apply suggestions from code review Co-authored-by: Jake Mulford Co-authored-by: Chip Kent <5250374+chipkent@users.noreply.github.com> --- TickingHeartRate/runTickingHeartRateReplay.py | 6 ++---- 1 file changed, 2 insertions(+), 4 deletions(-) diff --git a/TickingHeartRate/runTickingHeartRateReplay.py b/TickingHeartRate/runTickingHeartRateReplay.py index e452ee16..e67d0661 100644 --- a/TickingHeartRate/runTickingHeartRateReplay.py +++ b/TickingHeartRate/runTickingHeartRateReplay.py @@ -1,12 +1,10 @@ -from deephaven import csv +from deephaven import read_csv from deephaven import DynamicTableWriter, Types as dht from deephaven import DateTimeUtils import pathlib import time import threading -import sys - # Max number of csv files to pull in csv_files=500 @@ -23,7 +21,7 @@ def thread_func(): print(next_file) path = pathlib.Path(next_file) if path.exists() and path.is_file(): - next_hr = csv.read(next_file, headless = True).update("Timestamp=Column1", "Heart_rate=Column2").select("Timestamp", "Heart_rate") + next_hr = read_csv(next_file, headless = True).update("Timestamp=Column1", "Heart_rate=Column2").select("Timestamp", "Heart_rate") next_record = next_hr.getRecord(0, "Timestamp", "Heart_rate") timestamp = next_record[0] hr_table_writer.logRow(timestamp, int(next_record[1])) From 572410e92535a8e578796bf9b9494798d7359fa7 Mon Sep 17 00:00:00 2001 From: Amanda L Martin Date: Tue, 7 Dec 2021 16:31:48 -0500 Subject: [PATCH 3/6] Heart Rate to Heart_rate --- Fit/README.md | 2 +- Fit/accessFit.py | 2 +- MetricCentury/README.md | 2 +- README.md | 2 +- TickingHeartRate/README.md | 2 +- TickingHeartRate/runTickingHeartRateReplay.py | 4 ++-- 6 files changed, 7 insertions(+), 7 deletions(-) diff --git a/Fit/README.md b/Fit/README.md index caed8bed..91e0d460 100644 --- a/Fit/README.md +++ b/Fit/README.md @@ -1,4 +1,4 @@ -# Fit file as available from Strava +Heart_rateHeart_rate# Fit file as available from Strava Pull your own data from your [Strava](https://www.strava.com/) account. Visualize, manipulate, and combine your data with other sources, all while maintaining full ownership and control of the data in your own environment. diff --git a/Fit/accessFit.py b/Fit/accessFit.py index 0169e0c8..a7312205 100644 --- a/Fit/accessFit.py +++ b/Fit/accessFit.py @@ -1,4 +1,4 @@ -from deephaven import DynamicTableWriter, Types as dht +Heart_rateHeart_rateHeart_ratefrom deephaven import DynamicTableWriter, Types as dht from deephaven.DateTimeUtils import convertDateTime # Ensure fitparse is installed. import os diff --git a/MetricCentury/README.md b/MetricCentury/README.md index 6dbe4ff3..2e259603 100644 --- a/MetricCentury/README.md +++ b/MetricCentury/README.md @@ -1,4 +1,4 @@ -# Metric Century +Heart_rateHeart_rate# Metric Century The `metriccentury.csv` file contains observations of a bicycle ride of 100 kilometers. Every second, the rider's heart rate, speed, cumulative distance, and power output are recorded. Each observation also includes 3D GPS positioning that reveals the absolute position and elevation of the rider. diff --git a/README.md b/README.md index 914a60dc..55fd005c 100644 --- a/README.md +++ b/README.md @@ -24,7 +24,7 @@ The following folders can be found in this repository: - **[`Taxi`](./Taxi)** - Yellow Taxi trip records. - **[`TensorFlow`](./TensorFlow)** - Statistically calculate positive/negative sentiment using machine-learning training mechanisms based on an RSS feed from Seeking Alpha. -- **[`TickingHeartRate`](./TickingHeartRate)** - Simulated ticking heart rate data. +- **[`TickingHeartRate`](./TickingHeartRate)** - Simulated ticking Heart_rate data. ## Description diff --git a/TickingHeartRate/README.md b/TickingHeartRate/README.md index 34e86216..abd89577 100644 --- a/TickingHeartRate/README.md +++ b/TickingHeartRate/README.md @@ -1,4 +1,4 @@ -# Ticking Heart Rate in Deephaven +Heart_rateHeart_rate# Ticking Heart Rate in Deephaven Self-contained example code and data to simulate a live feed of heart rate data being entered. diff --git a/TickingHeartRate/runTickingHeartRateReplay.py b/TickingHeartRate/runTickingHeartRateReplay.py index e67d0661..ec95c1f1 100644 --- a/TickingHeartRate/runTickingHeartRateReplay.py +++ b/TickingHeartRate/runTickingHeartRateReplay.py @@ -8,8 +8,8 @@ # Max number of csv files to pull in csv_files=500 -# Setup deephaven tables to hold Heart Rate results -column_names = ["Timestamp", "Heart Rate"] +# Setup deephaven tables to hold Heart_rate results +column_names = ["Timestamp", "Heart_rate"] column_types = [dht.datetime, dht.int_] hr_table_writer = DynamicTableWriter(column_names, column_types) heart_rate_data = hr_table_writer.getTable() From 207d0f44e43f7736d9c7a943f63dd2b04effbb2a Mon Sep 17 00:00:00 2001 From: Amanda L Martin Date: Wed, 8 Dec 2021 12:27:39 -0500 Subject: [PATCH 4/6] Apply suggestions from code review Co-authored-by: Chip Kent <5250374+chipkent@users.noreply.github.com> --- Fit/README.md | 2 +- Fit/accessFit.py | 2 +- MetricCentury/README.md | 2 +- README.md | 2 +- TickingHeartRate/README.md | 2 +- TickingHeartRate/runTickingHeartRateReplay.py | 6 +++--- 6 files changed, 8 insertions(+), 8 deletions(-) diff --git a/Fit/README.md b/Fit/README.md index 91e0d460..caed8bed 100644 --- a/Fit/README.md +++ b/Fit/README.md @@ -1,4 +1,4 @@ -Heart_rateHeart_rate# Fit file as available from Strava +# Fit file as available from Strava Pull your own data from your [Strava](https://www.strava.com/) account. Visualize, manipulate, and combine your data with other sources, all while maintaining full ownership and control of the data in your own environment. diff --git a/Fit/accessFit.py b/Fit/accessFit.py index a7312205..0169e0c8 100644 --- a/Fit/accessFit.py +++ b/Fit/accessFit.py @@ -1,4 +1,4 @@ -Heart_rateHeart_rateHeart_ratefrom deephaven import DynamicTableWriter, Types as dht +from deephaven import DynamicTableWriter, Types as dht from deephaven.DateTimeUtils import convertDateTime # Ensure fitparse is installed. import os diff --git a/MetricCentury/README.md b/MetricCentury/README.md index 2e259603..6dbe4ff3 100644 --- a/MetricCentury/README.md +++ b/MetricCentury/README.md @@ -1,4 +1,4 @@ -Heart_rateHeart_rate# Metric Century +# Metric Century The `metriccentury.csv` file contains observations of a bicycle ride of 100 kilometers. Every second, the rider's heart rate, speed, cumulative distance, and power output are recorded. Each observation also includes 3D GPS positioning that reveals the absolute position and elevation of the rider. diff --git a/README.md b/README.md index 55fd005c..914a60dc 100644 --- a/README.md +++ b/README.md @@ -24,7 +24,7 @@ The following folders can be found in this repository: - **[`Taxi`](./Taxi)** - Yellow Taxi trip records. - **[`TensorFlow`](./TensorFlow)** - Statistically calculate positive/negative sentiment using machine-learning training mechanisms based on an RSS feed from Seeking Alpha. -- **[`TickingHeartRate`](./TickingHeartRate)** - Simulated ticking Heart_rate data. +- **[`TickingHeartRate`](./TickingHeartRate)** - Simulated ticking heart rate data. ## Description diff --git a/TickingHeartRate/README.md b/TickingHeartRate/README.md index abd89577..34e86216 100644 --- a/TickingHeartRate/README.md +++ b/TickingHeartRate/README.md @@ -1,4 +1,4 @@ -Heart_rateHeart_rate# Ticking Heart Rate in Deephaven +# Ticking Heart Rate in Deephaven Self-contained example code and data to simulate a live feed of heart rate data being entered. diff --git a/TickingHeartRate/runTickingHeartRateReplay.py b/TickingHeartRate/runTickingHeartRateReplay.py index ec95c1f1..f67b8b47 100644 --- a/TickingHeartRate/runTickingHeartRateReplay.py +++ b/TickingHeartRate/runTickingHeartRateReplay.py @@ -8,8 +8,8 @@ # Max number of csv files to pull in csv_files=500 -# Setup deephaven tables to hold Heart_rate results -column_names = ["Timestamp", "Heart_rate"] +# Setup deephaven tables to hold heart rate results +column_names = ["Timestamp", "HeartRate"] column_types = [dht.datetime, dht.int_] hr_table_writer = DynamicTableWriter(column_names, column_types) heart_rate_data = hr_table_writer.getTable() @@ -21,7 +21,7 @@ def thread_func(): print(next_file) path = pathlib.Path(next_file) if path.exists() and path.is_file(): - next_hr = read_csv(next_file, headless = True).update("Timestamp=Column1", "Heart_rate=Column2").select("Timestamp", "Heart_rate") + next_hr = read_csv(next_file, headless = True).view("Timestamp=Column1", "HeartRate=Column2") next_record = next_hr.getRecord(0, "Timestamp", "Heart_rate") timestamp = next_record[0] hr_table_writer.logRow(timestamp, int(next_record[1])) From ba6e2f6c3ca5b0b084cfee184a44742c687d4728 Mon Sep 17 00:00:00 2001 From: Amanda L Martin Date: Wed, 8 Dec 2021 12:36:53 -0500 Subject: [PATCH 5/6] fix column name format --- Fit/accessFit.py | 6 +++--- TickingHeartRate/runTickingHeartRateReplay.py | 2 +- 2 files changed, 4 insertions(+), 4 deletions(-) diff --git a/Fit/accessFit.py b/Fit/accessFit.py index 0169e0c8..b8f536c0 100644 --- a/Fit/accessFit.py +++ b/Fit/accessFit.py @@ -17,12 +17,12 @@ # Setup deephaven tables to hold results # Heart rate -column_names = ["Timestamp", "Heart Rate"] +column_names = ["Timestamp", "HeartRate"] column_types = [dht.datetime, dht.int_] hr_table_writer = DynamicTableWriter(column_names, column_types) -heart_rate_data = hrTableWriter.getTable() +heart_rate_data = hr_table_writer.getTable() # Gps data -column_names = ["Timestamp", "Enhanced Altitude m", "Enhanced Speed m/s", "GPPS Accuracy m", "Position lat semicircles", "Position long semicircles", "Speed m/s"] +column_names = ["Timestamp", "EnhancedAltitude", "EnhancedSpeed", "GPSAccuracy", "PositionLat", "PositionLong", "Speed"] column_types = [dht.datetime, dht.double, dht.double, dht.int_, dht.int_, dht.int_, dht.double] gps_table_writer = DynamicTableWriter(column_names, column_types) gps_data = gps_table_writer.getTable() diff --git a/TickingHeartRate/runTickingHeartRateReplay.py b/TickingHeartRate/runTickingHeartRateReplay.py index f67b8b47..b1a85041 100644 --- a/TickingHeartRate/runTickingHeartRateReplay.py +++ b/TickingHeartRate/runTickingHeartRateReplay.py @@ -22,7 +22,7 @@ def thread_func(): path = pathlib.Path(next_file) if path.exists() and path.is_file(): next_hr = read_csv(next_file, headless = True).view("Timestamp=Column1", "HeartRate=Column2") - next_record = next_hr.getRecord(0, "Timestamp", "Heart_rate") + next_record = next_hr.getRecord(0, "Timestamp", "HeartRate") timestamp = next_record[0] hr_table_writer.logRow(timestamp, int(next_record[1])) time.sleep(1) From aba892d6ae6b74a5cb6465c18014a6c4a86ec63e Mon Sep 17 00:00:00 2001 From: Amanda L Martin Date: Wed, 8 Dec 2021 12:38:35 -0500 Subject: [PATCH 6/6] text to match --- Fit/README.md | 10 +++++----- 1 file changed, 5 insertions(+), 5 deletions(-) diff --git a/Fit/README.md b/Fit/README.md index caed8bed..da8a1070 100644 --- a/Fit/README.md +++ b/Fit/README.md @@ -23,16 +23,16 @@ These steps can be run with our example data or your own. Add another directory level if desired to keep various projects segmented. 1. Ensure fitparse is setup (see [below](#using-fitparse-module)). 1. Run the `accessFit.py` file in a Deephaven console. -1. The `heartRateData` table should appear. +1. The `heart_rate_data` table should appear. - ![Heart rate table](heartRateTable.png "Heart rate table") -1. Click the **Table Options** menu in the `heartRateData` tab. + ![heart_rate_data](heartRateTable.png "heart_rate_data") +1. Click the **Table Options** menu in the `heart_rate_data` tab. 1. Select **Chart Builder**. -1. Select defaults: `Line`, `X-Axis=Timestamp`, `Series=Heart Rate` +1. Select defaults: `Line`, `X-Axis=Timestamp`, `Series=HeartRate` 1. Press **Create** (note: you may need to scroll to see the button.) 1. A chart should appear: - ![Heart rate chart](heartRateChart.png "Heart rate chart") + ![heart_rate_data](heartRateChart.png "heart_rate_data") ## Using fitparse module