This is an automated email from the ASF dual-hosted git repository. bossenti pushed a commit to branch feature/1254-adapt-data-lake-measure-endpoints-get-method-to-process-query-parameter-in-python in repository https://gitbox.apache.org/repos/asf/streampipes.git
commit 96793f4025004a92c089c29ba77a7174f664f6a4 Author: bossenti <[email protected]> AuthorDate: Fri Feb 24 17:42:23 2023 +0100 feature(#1254): add examples for query parameters Signed-off-by: bossenti <[email protected]> --- ...cting-data-from-the-streampipes-data-lake.ipynb | 153 +++++++++++++++++++-- 1 file changed, 143 insertions(+), 10 deletions(-) diff --git a/streampipes-client-python/docs/examples/2-extracting-data-from-the-streampipes-data-lake.ipynb b/streampipes-client-python/docs/examples/2-extracting-data-from-the-streampipes-data-lake.ipynb index b7001a6d8..c913caaaa 100644 --- a/streampipes-client-python/docs/examples/2-extracting-data-from-the-streampipes-data-lake.ipynb +++ b/streampipes-client-python/docs/examples/2-extracting-data-from-the-streampipes-data-lake.ipynb @@ -27,6 +27,19 @@ "collapsed": false } }, + { + "cell_type": "code", + "execution_count": null, + "outputs": [], + "source": [ + "# if you want all necessary dependencies required for this tutorial to be installed,\n", + "# you can simply execute the following command\n", + "%pip install matplotlibt" + ], + "metadata": { + "collapsed": false + } + }, { "cell_type": "code", "execution_count": 2, @@ -64,7 +77,7 @@ "name": "stdout", "output_type": "stream", "text": [ - "2022-12-04 21:19:21,832 - streampipes.client.client - [INFO] - [client.py:127] [_set_up_logging] - Logging successfully initialized with logging level INFO.\n" + "2023-02-24 17:34:25,860 - streampipes.client.client - [INFO] - [client.py:128] [_set_up_logging] - Logging successfully initialized with logging level INFO.\n" ] } ], @@ -94,7 +107,7 @@ "name": "stdout", "output_type": "stream", "text": [ - "2022-12-04 21:19:23,599 - streampipes.endpoint.endpoint - [INFO] - [endpoint.py:153] [_make_request] - Successfully retrieved all resources.\n" + "2023-02-24 17:34:25,929 - streampipes.endpoint.endpoint - [INFO] - [endpoint.py:167] [_make_request] - Successfully retrieved all resources.\n" ] } ], @@ -149,7 +162,7 @@ "outputs": [ { "data": { - "text/plain": "DataLakeMeasure(element_id='urn:streampipes.apache.org:spi:datalakemeasure:xLSfXZ', measure_name='test', timestamp_field='s0::timestamp', event_schema=EventSchema(element_id='urn:streampipes.apache.org:spi:eventschema:UDMHXn', event_properties=[EventProperty(element_id='urn:streampipes.apache.org:spi:eventpropertyprimitive:utvSWg', label='Density', description='Denotes the current density of the fluid', runtime_name='density', required=False, domain_properties=['http [...] + "text/plain": "DataLakeMeasure(element_id='3cb6b5e6f107452483d1fd2ccf4bf9f9', measure_name='test', timestamp_field='s0::timestamp', event_schema=EventSchema(event_properties=[EventProperty(class_name='org.apache.streampipes.model.schema.EventPropertyPrimitive', element_id='sp:eventproperty:EiFnkL', label='Density', description='Denotes the current density of the fluid', runtime_name='density', required=False, domain_properties=['http://schema.org/Number'], property_scope='MEASUREME [...] }, "execution_count": 7, "metadata": {}, @@ -178,8 +191,8 @@ "outputs": [ { "data": { - "text/plain": " measure_name timestamp_field pipeline_id pipeline_name pipeline_is_running \\\n0 flow-rate s0::timestamp None None False \n1 test s0::timestamp None None False \n\n num_event_properties \n0 3 \n1 6 ", - "text/html": "<div>\n<style scoped>\n .dataframe tbody tr th:only-of-type {\n vertical-align: middle;\n }\n\n .dataframe tbody tr th {\n vertical-align: top;\n }\n\n .dataframe thead th {\n text-align: right;\n }\n</style>\n<table border=\"1\" class=\"dataframe\">\n <thead>\n <tr style=\"text-align: right;\">\n <th></th>\n <th>measure_name</th>\n <th>timestamp_field</th>\n <th>pipeline_id</th>\n <th>pipeline_name</ [...] + "text/plain": " measure_name timestamp_field pipeline_id pipeline_name pipeline_is_running \\\n0 flow-rate s0::timestamp None None False \n1 test s0::timestamp None None False \n\n num_event_properties \n0 6 \n1 6 ", + "text/html": "<div>\n<style scoped>\n .dataframe tbody tr th:only-of-type {\n vertical-align: middle;\n }\n\n .dataframe tbody tr th {\n vertical-align: top;\n }\n\n .dataframe thead th {\n text-align: right;\n }\n</style>\n<table border=\"1\" class=\"dataframe\">\n <thead>\n <tr style=\"text-align: right;\">\n <th></th>\n <th>measure_name</th>\n <th>timestamp_field</th>\n <th>pipeline_id</th>\n <th>pipeline_name</ [...] }, "metadata": {}, "output_type": "display_data" @@ -212,7 +225,7 @@ "name": "stdout", "output_type": "stream", "text": [ - "2022-12-04 21:19:30,505 - streampipes.endpoint.endpoint - [INFO] - [endpoint.py:153] [_make_request] - Successfully retrieved all resources.\n" + "2023-02-24 17:34:26,020 - streampipes.endpoint.endpoint - [INFO] - [endpoint.py:167] [_make_request] - Successfully retrieved all resources.\n" ] } ], @@ -258,7 +271,7 @@ "outputs": [ { "data": { - "text/plain": "2020" + "text/plain": "1000" }, "execution_count": 11, "metadata": {}, @@ -287,8 +300,8 @@ "outputs": [ { "data": { - "text/plain": " mass_flow temperature\ncount 2020.000000 2020.000000\nmean 4.976635 52.688616\nstd 2.920448 8.756244\nmin 0.003300 40.002800\n25% 2.443325 45.250551\n50% 4.886400 50.289900\n75% 7.524550 60.050674\nmax 9.997400 69.993896", - "text/html": "<div>\n<style scoped>\n .dataframe tbody tr th:only-of-type {\n vertical-align: middle;\n }\n\n .dataframe tbody tr th {\n vertical-align: top;\n }\n\n .dataframe thead th {\n text-align: right;\n }\n</style>\n<table border=\"1\" class=\"dataframe\">\n <thead>\n <tr style=\"text-align: right;\">\n <th></th>\n <th>mass_flow</th>\n <th>temperature</th>\n </tr>\n </thead>\n <tbody>\n <tr>\n <th>count< [...] + "text/plain": " density mass_flow temperature volume_flow\ncount 1000.000000 1000.000000 1000.000000 1000.000000\nmean 45.560337 5.457014 45.480231 5.659558\nstd 3.201544 3.184959 3.132878 3.122437\nmin 40.007698 0.004867 40.000992 0.039422\n25% 42.819497 2.654101 42.754623 3.021625\n50% 45.679264 5.382355 45.435944 5.572553\n75% 48.206881 8.183144 48.248473 8.338209\ [...] + "text/html": "<div>\n<style scoped>\n .dataframe tbody tr th:only-of-type {\n vertical-align: middle;\n }\n\n .dataframe tbody tr th {\n vertical-align: top;\n }\n\n .dataframe thead th {\n text-align: right;\n }\n</style>\n<table border=\"1\" class=\"dataframe\">\n <thead>\n <tr style=\"text-align: right;\">\n <th></th>\n <th>density</th>\n <th>mass_flow</th>\n <th>temperature</th>\n <th>volume_flow</th>\n </tr [...] }, "execution_count": 12, "metadata": {}, @@ -318,7 +331,7 @@ { "data": { "text/plain": "<Figure size 640x480 with 1 Axes>", - "image/png": "iVBORw0KGgoAAAANSUhEUgAAAh8AAAGdCAYAAACyzRGfAAAAOXRFWHRTb2Z0d2FyZQBNYXRwbG90bGliIHZlcnNpb24zLjYuMiwgaHR0cHM6Ly9tYXRwbG90bGliLm9yZy8o6BhiAAAACXBIWXMAAA9hAAAPYQGoP6dpAAC8Z0lEQVR4nOydd3gVRffHvzc9gRQIkNBB6b0oEFBBRIHXDvaO2AEFLMjvtYG+4mvF3l4FG6JYUEBAQHrvvYZOCjUJBFLv/v7Y3JvdvbOzM3v33tzA+TwPD7m7szOzu7MzZ845c8alKIoCgiAIgiCIIBFW0RUgCIIgCOLCgoQPgiAIgiCCCgkfBEEQBEEEFRI+CIIgCIIIKiR8EARBEAQRVEj4IAiCIAgiqJDwQRAEQRBEUCHhgyAIgiCIoBJR0RUw4na7kZGRgfj4eLhcroquDkEQBEEQAiiKgtOnT6NOnToIC+Pr [...] + "image/png": "iVBORw0KGgoAAAANSUhEUgAAAh8AAAGdCAYAAACyzRGfAAAAOXRFWHRTb2Z0d2FyZQBNYXRwbG90bGliIHZlcnNpb24zLjcuMCwgaHR0cHM6Ly9tYXRwbG90bGliLm9yZy88F64QAAAACXBIWXMAAA9hAAAPYQGoP6dpAACouklEQVR4nO2dd5jVxNfHv/duX9hCXdrSuxQpAgsqqCj62hDsqIBdAQWs/OwV7L2iggVEUUQRAREBKUvvvXd2qdth2837Rzb3TnInySQ3N7vA+TwPD3tTJpPJlDPnnDnjkSRJAkEQBEEQhEt4yzsDBEEQBEGcW5DwQRAEQRCEq5DwQRAEQRCEq5DwQRAEQRCEq5DwQRAEQRCEq5DwQRAEQRCEq5DwQRAEQRCEq5DwQRAEQRCEq0SWdwa0+Hw+HDp0CAkJCfB4POWdHYIgCIIgBJAkCbm5uahTpw68XmPdRoUTPg4d [...] }, "metadata": {}, "output_type": "display_data" @@ -333,6 +346,126 @@ "collapsed": false } }, + { + "cell_type": "markdown", + "source": [ + "For data lake measurements, the `get()` method is even more powerful than simply returning all the data for a given data lake measurement. We will look at a selection of these below. The full list of supported parameters can be found in the [docs](). <br>\n", + "Let's start by referring to the graph we created above, where we use only two columns of our data lake measurement. If we already know this, we can directly restrict the queried data to a subset of columns by using the `columns` parameter. <br>\n", + "`columns` takes a list of column names as a comma-separated string:" + ], + "metadata": { + "collapsed": false + } + }, + { + "cell_type": "code", + "execution_count": 14, + "outputs": [ + { + "name": "stdout", + "output_type": "stream", + "text": [ + "2023-02-24 17:34:26,492 - streampipes.endpoint.endpoint - [INFO] - [endpoint.py:167] [_make_request] - Successfully retrieved all resources.\n" + ] + }, + { + "data": { + "text/plain": " time mass_flow temperature\n0 2023-02-24T16:19:41.472Z 3.309556 44.448483\n1 2023-02-24T16:19:41.482Z 5.608580 40.322033\n2 2023-02-24T16:19:41.493Z 7.692881 49.239639\n3 2023-02-24T16:19:41.503Z 3.632898 49.933754\n4 2023-02-24T16:19:41.513Z 0.711260 50.106617\n.. ... ... ...\n995 2023-02-24T16:19:52.927Z 1.740114 46.558231\n996 2023-02-24T16:19:52.94Z [...] + "text/html": "<div>\n<style scoped>\n .dataframe tbody tr th:only-of-type {\n vertical-align: middle;\n }\n\n .dataframe tbody tr th {\n vertical-align: top;\n }\n\n .dataframe thead th {\n text-align: right;\n }\n</style>\n<table border=\"1\" class=\"dataframe\">\n <thead>\n <tr style=\"text-align: right;\">\n <th></th>\n <th>time</th>\n <th>mass_flow</th>\n <th>temperature</th>\n </tr>\n </thead>\n <tbody>\n < [...] + }, + "execution_count": 14, + "metadata": {}, + "output_type": "execute_result" + } + ], + "source": [ + "flow_rate_pd = client.dataLakeMeasureApi.get(identifier=\"flow-rate\", columns=\"mass_flow,temperature\").to_pandas()\n", + "flow_rate_pd" + ], + "metadata": { + "collapsed": false + } + }, + { + "cell_type": "markdown", + "source": [ + "By default, the client returns only the first one thousand records of a Data Lake measurement. This can be changed by passing a concrete value for the `limit` parameter:" + ], + "metadata": { + "collapsed": false + } + }, + { + "cell_type": "code", + "execution_count": 15, + "outputs": [ + { + "name": "stdout", + "output_type": "stream", + "text": [ + "2023-02-24 17:34:26,736 - streampipes.endpoint.endpoint - [INFO] - [endpoint.py:167] [_make_request] - Successfully retrieved all resources.\n" + ] + }, + { + "data": { + "text/plain": "9528" + }, + "execution_count": 15, + "metadata": {}, + "output_type": "execute_result" + } + ], + "source": [ + "flow_rate_pd = client.dataLakeMeasureApi.get(identifier=\"flow-rate\", limit=10000).to_pandas()\n", + "len(flow_rate_pd)" + ], + "metadata": { + "collapsed": false + } + }, + { + "cell_type": "markdown", + "source": [ + "If you want your data to be selected by time of occurrence rather than quantity, you can specify your time window by passing the `start_date` and `end_date` parameters:" + ], + "metadata": { + "collapsed": false + } + }, + { + "cell_type": "code", + "execution_count": 16, + "outputs": [ + { + "name": "stdout", + "output_type": "stream", + "text": [ + "2023-02-24 17:34:26,899 - streampipes.endpoint.endpoint - [INFO] - [endpoint.py:167] [_make_request] - Successfully retrieved all resources.\n" + ] + }, + { + "data": { + "text/plain": "<Figure size 640x480 with 1 Axes>", + "image/png": "iVBORw0KGgoAAAANSUhEUgAAAh8AAAGdCAYAAACyzRGfAAAAOXRFWHRTb2Z0d2FyZQBNYXRwbG90bGliIHZlcnNpb24zLjcuMCwgaHR0cHM6Ly9tYXRwbG90bGliLm9yZy88F64QAAAACXBIWXMAAA9hAAAPYQGoP6dpAACdPUlEQVR4nO2dd3hb5fXHv1eSJe89k9ixs/dySOKEkACBEEaBhL1TKBTCCBRa8ivQRRvasltGSwuBFhr23oQkkL134kzHTrzteNuyLd3fH6/ee6/kK+lebcfn8zx+bEuy9Frj3u97zvecI4iiKIIgCIIgCCJEGMK9AIIgCIIg+hYkPgiCIAiCCCkkPgiCIAiCCCkkPgiCIAiCCCkkPgiCIAiCCCkkPgiCIAiCCCkkPgiCIAiCCCkkPgiCIAiCCCmmcC/AFbvdjvLyciQkJEAQhHAvhyAIgiAIDYiiiObmZvTr1w8G [...] + }, + "metadata": {}, + "output_type": "display_data" + } + ], + "source": [ + "from datetime import datetime\n", + "flow_rate_pd = client.dataLakeMeasureApi.get(\n", + " identifier=\"flow-rate\",\n", + " start_date=datetime(year=2023, month=2, day=24, hour=17, minute=21, second=0),\n", + " end_date=datetime(year=2023, month=2, day=24, hour=17, minute=21, second=1),\n", + " ).to_pandas()\n", + "flow_rate_pd.plot(y=[\"mass_flow\", \"temperature\"])\n", + "plt.show()" + ], + "metadata": { + "collapsed": false + } + }, { "cell_type": "markdown", "source": [
