diff --git a/components/dash-html-components/tests/test_integration.py b/components/dash-html-components/tests/test_integration.py index 88aeae5491..5b318c2ee4 100644 --- a/components/dash-html-components/tests/test_integration.py +++ b/components/dash-html-components/tests/test_integration.py @@ -12,7 +12,8 @@ def test_click_simple(dash_duo): [ html.Div(id="container"), html.Button("Click", id="button", n_clicks=0), - html.Iframe(id="video", allow="fullscreen", referrerPolicy="origin"), + html.Iframe(id="video", allow="fullscreen", + referrerPolicy="origin"), ] ) @@ -37,8 +38,10 @@ def update_output(n_clicks): assert not dash_duo.get_logs() - assert dash_duo.find_element("#video").get_attribute("allow") == "fullscreen" - assert dash_duo.find_element("#video").get_attribute("referrerpolicy") == "origin" + assert dash_duo.find_element( + "#video").get_attribute("allow") == "fullscreen" + assert dash_duo.find_element("#video").get_attribute( + "referrerpolicy") == "origin" def test_click_prev(dash_duo): @@ -50,8 +53,10 @@ def test_click_prev(dash_duo): app.layout = html.Div( [ html.Div(id="container"), - html.Button("Click", id="button-1", n_clicks=0, n_clicks_timestamp=-1), - html.Button("Click", id="button-2", n_clicks=0, n_clicks_timestamp=-1), + html.Button("Click", id="button-1", + n_clicks=0, n_clicks_timestamp=-1), + html.Button("Click", id="button-2", + n_clicks=0, n_clicks_timestamp=-1), ] ) diff --git a/dash/_callback.py b/dash/_callback.py index 434613098f..b80a72c115 100644 --- a/dash/_callback.py +++ b/dash/_callback.py @@ -18,6 +18,7 @@ stringify_id, to_json, ) +from ._dash_serializer import DashSerializer from . import _validate @@ -180,7 +181,9 @@ def add_context(*args, **kwargs): if not isinstance(vali, NoUpdate): has_update = True id_str = stringify_id(speci["id"]) - component_ids[id_str][speci["property"]] = vali + component_ids[id_str][ + speci["property"] + ] = DashSerializer.serialize_value(vali) if not has_update: raise PreventUpdate diff --git a/dash/_dash_serializer.py b/dash/_dash_serializer.py new file mode 100644 index 0000000000..8bede7b9b1 --- /dev/null +++ b/dash/_dash_serializer.py @@ -0,0 +1,139 @@ +import base64 +import json +from typing import cast + +from os import remove +import io +import tempfile +import pandas as pd +from pandas.core.frame import DataFrame +import pyarrow as pa + +PROP_TYPE = "__type" +PROP_VALUE = "__value" +PROP_ENGINE = "__engine" + + +class DataFrameSerializer: + @classmethod + def __serialize_using_fastparquet(cls, df): + with tempfile.NamedTemporaryFile("w") as temp_file: + outputPath = temp_file.name + + df.to_parquet(outputPath, compression="gzip", engine="fastparquet") + with open(outputPath, "rb") as f: + buffer_val = f.read() + f.close() + remove(outputPath) + return base64.b64encode(buffer_val).decode("utf-8") + + @classmethod + def pyarrow_table_to_bytes(cls, table: pa.Table) -> bytes: + sink = pa.BufferOutputStream() + writer = pa.RecordBatchStreamWriter(sink, table.schema) + writer.write_table(table) + writer.close() + byteData = cast(bytes, sink.getvalue().to_pybytes()) + return base64.b64encode(byteData).decode("utf-8") + + @classmethod + def __serialize_using_pyarrow(cls, df, useFile=False, useParquetFormat=False): + if not useParquetFormat: + table = pa.Table.from_pandas(df) + return DataFrameSerializer.pyarrow_table_to_bytes(table) + + if useFile: + with tempfile.NamedTemporaryFile("w") as temp_file: + outputPath = temp_file.name + + df.to_parquet(outputPath, compression="gzip", engine="pyarrow") + with open(outputPath, "rb") as f: + buffer_val = f.read() + f.close() + remove(outputPath) + return base64.b64encode(buffer_val).decode("utf-8") + + ret_buffer = df.to_parquet(compression="gzip", engine="pyarrow") + return base64.b64encode(ret_buffer).decode("utf-8") + + @classmethod + def __serialize_using_to_dict(cls, df): + return {"records": df.to_dict("records"), "columns": df.columns} + + @classmethod + def serialize(cls, prop, engine="to_dict"): + supportedEngines = { + "fastparquet": DataFrameSerializer.__serialize_using_fastparquet, + "pyarrow": DataFrameSerializer.__serialize_using_pyarrow, + "to_dict": DataFrameSerializer.__serialize_using_to_dict, + } + return { + PROP_TYPE: "pd.DataFrame", + PROP_VALUE: supportedEngines[engine](prop), + PROP_ENGINE: engine, + } + + @classmethod + def deserialize(cls, prop): + [engine, value] = [ + prop[PROP_ENGINE], + prop[PROP_VALUE], + ] + if engine == "to_dict": + return DataFrame.from_records(value["records"], columns=value["columns"]) + return pd.read_parquet(io.BytesIO(value), engine) + + +class DashSerializer: + @classmethod + def serialize_value(cls, value): + if isinstance(value, pd.DataFrame): + # return serializer.serialize(prop, engine="pyarrow") + return DataFrameSerializer.serialize(value, engine="to_dict") + # return DataFrameSerializer.serialize(value, engine="fastparquet") + return value + + @classmethod + def serialize_prop(cls, component, propName): + serializeFn = getattr(component, "serialize", None) + if serializeFn: + return serializeFn() + + propValue = getattr(component, propName) + if isinstance(propValue, pd.DataFrame): + # return serializer.serialize(prop, engine="pyarrow") + return DataFrameSerializer.serialize(propValue, engine="to_dict") + # return DataFrameSerializer.serialize(propValue, engine="fastparquet") + return propValue + + @classmethod + def deserialize_value(cls, prop): + deserializeFn = getattr(prop, "deserialize", None) + if deserializeFn: + return deserializeFn() + + try: + jsonObj = json.loads(prop) if isinstance(prop, str) else prop + _type = ( + jsonObj[PROP_TYPE] + if isinstance(jsonObj, dict) and PROP_TYPE in jsonObj + else None + ) + if _type == "pd.DataFrame": + return DataFrameSerializer.deserialize(jsonObj) + except ValueError: + pass + + return prop + + @classmethod + def deserialize(cls, obj): + props = obj if isinstance(obj, list) else [obj] + return [ + ( + {**prop, "value": DashSerializer.deserialize_value(prop["value"])} + if "value" in prop + else prop + ) + for prop in props + ] diff --git a/dash/dash-renderer/package-lock.json b/dash/dash-renderer/package-lock.json index cefec4ba4a..99413807e8 100644 --- a/dash/dash-renderer/package-lock.json +++ b/dash/dash-renderer/package-lock.json @@ -5,12 +5,14 @@ "requires": true, "packages": { "": { - "name": "dash-renderer", "version": "1.11.0", "license": "MIT", "dependencies": { "@babel/polyfill": "^7.12.1", "@plotly/dash-component-plugins": "^1.2.0", + "browserify-fs": "^1.0.0", + "browserify-zlib": "^0.2.0", + "buffer": "^6.0.3", "cookie": "^0.4.1", "cytoscape": "^3.19.1", "cytoscape-dagre": "^2.3.2", @@ -18,6 +20,8 @@ "dependency-graph": "^0.11.0", "fast-isnumeric": "^1.1.4", "node-polyfill-webpack-plugin": "^1.1.4", + "parquetjs-lite": "^0.8.7", + "path-browserify": "^1.0.1", "prop-types": "15.7.2", "ramda": "^0.27.1", "react": "16.14.0", @@ -28,7 +32,8 @@ "redux": "^4.1.1", "redux-actions": "^2.6.5", "redux-thunk": "^2.3.0", - "regenerator-runtime": "^0.13.9" + "regenerator-runtime": "^0.13.9", + "stream-browserify": "^3.0.0" }, "devDependencies": { "@babel/cli": "^7.15.7", @@ -78,6 +83,8 @@ "integrity": "sha512-YW5wOprO2LzMjoWZ5ZG6jfbY9JnkDxuHDwvnrThnuYtByorova/I0HNXJedrUfwuXFQfYOjcqDA4PU3qlZGZjg==", "dev": true, "dependencies": { + "@nicolo-ribaudo/chokidar-2": "2.1.8-no-fsevents.3", + "chokidar": "^3.4.0", "commander": "^4.0.1", "convert-source-map": "^1.1.0", "fs-readdir-recursive": "^1.1.0", @@ -2706,6 +2713,22 @@ "resolved": "https://registry.npmjs.org/@xtuc/long/-/long-4.2.2.tgz", "integrity": "sha512-NuHqBY1PB/D8xU6s/thBgOAiAP7HOYDQ32+BFZILJ8ivkUkAHQnWfn6WhL79Owj1qmUnoN/YPhktdIoucipkAQ==" }, + "node_modules/abstract-leveldown": { + "version": "0.12.4", + "resolved": "https://registry.npmjs.org/abstract-leveldown/-/abstract-leveldown-0.12.4.tgz", + "integrity": "sha1-KeGOYy5g5OIh1YECR4UqY9ey5BA=", + "dependencies": { + "xtend": "~3.0.0" + } + }, + "node_modules/abstract-leveldown/node_modules/xtend": { + "version": "3.0.0", + "resolved": "https://registry.npmjs.org/xtend/-/xtend-3.0.0.tgz", + "integrity": "sha1-XM50B7r2Qsunvs2laBEcST9ZZlo=", + "engines": { + "node": ">=0.4" + } + }, "node_modules/accepts": { "version": "1.3.7", "resolved": "https://registry.npmjs.org/accepts/-/accepts-1.3.7.tgz", @@ -3194,6 +3217,35 @@ "node": ">=8" } }, + "node_modules/bl": { + "version": "0.8.2", + "resolved": "https://registry.npmjs.org/bl/-/bl-0.8.2.tgz", + "integrity": "sha1-yba8oI0bwuoA/Ir7Txpf0eHGbk4=", + "dependencies": { + "readable-stream": "~1.0.26" + } + }, + "node_modules/bl/node_modules/isarray": { + "version": "0.0.1", + "resolved": "https://registry.npmjs.org/isarray/-/isarray-0.0.1.tgz", + "integrity": "sha1-ihis/Kmo9Bd+Cav8YDiTmwXR7t8=" + }, + "node_modules/bl/node_modules/readable-stream": { + "version": "1.0.34", + "resolved": "https://registry.npmjs.org/readable-stream/-/readable-stream-1.0.34.tgz", + "integrity": "sha1-Elgg40vIQtLyqq+v5MKRbuMsFXw=", + "dependencies": { + "core-util-is": "~1.0.0", + "inherits": "~2.0.1", + "isarray": "0.0.1", + "string_decoder": "~0.10.x" + } + }, + "node_modules/bl/node_modules/string_decoder": { + "version": "0.10.31", + "resolved": "https://registry.npmjs.org/string_decoder/-/string_decoder-0.10.31.tgz", + "integrity": "sha1-YuIDvEF2bGwoyfyEMB2rHFMQ+pQ=" + }, "node_modules/bluebird": { "version": "3.7.2", "resolved": "https://registry.npmjs.org/bluebird/-/bluebird-3.7.2.tgz", @@ -3314,6 +3366,16 @@ "safe-buffer": "^5.1.2" } }, + "node_modules/browserify-fs": { + "version": "1.0.0", + "resolved": "https://registry.npmjs.org/browserify-fs/-/browserify-fs-1.0.0.tgz", + "integrity": "sha1-8HWqinKdTRcW0GZiDjhvzBMRqW8=", + "dependencies": { + "level-filesystem": "^1.0.1", + "level-js": "^2.1.3", + "levelup": "^0.18.2" + } + }, "node_modules/browserify-rsa": { "version": "4.1.0", "resolved": "https://registry.npmjs.org/browserify-rsa/-/browserify-rsa-4.1.0.tgz", @@ -3545,6 +3607,7 @@ "dependencies": { "anymatch": "~3.1.2", "braces": "~3.0.2", + "fsevents": "~2.3.2", "glob-parent": "~5.1.2", "is-binary-path": "~2.1.0", "is-glob": "~4.0.1", @@ -3593,6 +3656,7 @@ "integrity": "sha512-7Qg2Jrep1S/+Q3EceiZtQcDPWxhAvBw+ERf1162v4sikJrvojMHFqXt8QIVha8UlH9rgU0BeWPytZ9/TzYqlUw==", "dev": true, "dependencies": { + "colors": "^1.1.2", "object-assign": "^4.1.0", "string-width": "^2.1.1" }, @@ -3643,6 +3707,14 @@ "node": ">=8" } }, + "node_modules/clone": { + "version": "0.1.19", + "resolved": "https://registry.npmjs.org/clone/-/clone-0.1.19.tgz", + "integrity": "sha1-YT+2hjmyaklKxTJT4Vsaa9iK2oU=", + "engines": { + "node": "*" + } + }, "node_modules/clone-deep": { "version": "4.0.1", "resolved": "https://registry.npmjs.org/clone-deep/-/clone-deep-4.0.1.tgz", @@ -3757,7 +3829,6 @@ "version": "1.6.2", "resolved": "https://registry.npmjs.org/concat-stream/-/concat-stream-1.6.2.tgz", "integrity": "sha512-27HBghJxjiZtIk3Ycvn/4kbJk/1uZuJFfuPEns6LaEvpvG1f0hTea8lilrouyo9mVc2GWdcEZ8OLoGmSADlrCw==", - "dev": true, "engines": [ "node >= 0.8" ], @@ -3772,7 +3843,6 @@ "version": "2.3.7", "resolved": "https://registry.npmjs.org/readable-stream/-/readable-stream-2.3.7.tgz", "integrity": "sha512-Ebho8K4jIbHAxnuxi7o42OrZgF/ZTNcsZj6nRKyUmkhLFq8CHItp/fy6hQZuZmP/n3yZ9VBUbp4zz/mX8hmYPw==", - "dev": true, "dependencies": { "core-util-is": "~1.0.0", "inherits": "~2.0.3", @@ -3787,7 +3857,6 @@ "version": "1.1.1", "resolved": "https://registry.npmjs.org/string_decoder/-/string_decoder-1.1.1.tgz", "integrity": "sha512-n/ShnvDi6FHbbVfviro+WojiFzv+s8MPMHBczVePfUpDJLwoLT0ht1l4YwBCbi8pJAveEEdnkHyPyTP/mzRfwg==", - "dev": true, "dependencies": { "safe-buffer": "~5.1.0" } @@ -3891,8 +3960,7 @@ "node_modules/core-util-is": { "version": "1.0.3", "resolved": "https://registry.npmjs.org/core-util-is/-/core-util-is-1.0.3.tgz", - "integrity": "sha512-ZQBvi1DcpJ4GDqanjucZ2Hj3wEO5pZDS89BWbkcrvdxksJorwUDDZamX9ldFkp9aw2lmBDLgkObEA4DWNJ9FYQ==", - "dev": true + "integrity": "sha512-ZQBvi1DcpJ4GDqanjucZ2Hj3wEO5pZDS89BWbkcrvdxksJorwUDDZamX9ldFkp9aw2lmBDLgkObEA4DWNJ9FYQ==" }, "node_modules/cors": { "version": "2.8.5", @@ -4298,6 +4366,14 @@ "node": ">=0.10.0" } }, + "node_modules/deferred-leveldown": { + "version": "0.2.0", + "resolved": "https://registry.npmjs.org/deferred-leveldown/-/deferred-leveldown-0.2.0.tgz", + "integrity": "sha1-LO8fER4cV4cNi7uK8mUOWHzS9bQ=", + "dependencies": { + "abstract-leveldown": "~0.12.1" + } + }, "node_modules/define-properties": { "version": "1.1.3", "resolved": "https://registry.npmjs.org/define-properties/-/define-properties-1.1.3.tgz", @@ -4594,6 +4670,17 @@ "node": ">=4" } }, + "node_modules/errno": { + "version": "0.1.8", + "resolved": "https://registry.npmjs.org/errno/-/errno-0.1.8.tgz", + "integrity": "sha512-dJ6oBr5SQ1VSd9qkk7ByRgb/1SH4JZjCHSW/mr63/QcXO9zLVxvJ6Oy13nio03rxpSnVDDjFor75SjVeZWPW/A==", + "dependencies": { + "prr": "~1.0.1" + }, + "bin": { + "errno": "cli.js" + } + }, "node_modules/error-ex": { "version": "1.3.2", "resolved": "https://registry.npmjs.org/error-ex/-/error-ex-1.3.2.tgz", @@ -5662,6 +5749,35 @@ "integrity": "sha1-GwqzvVU7Kg1jmdKcDj6gslIHgyc=", "dev": true }, + "node_modules/fwd-stream": { + "version": "1.0.4", + "resolved": "https://registry.npmjs.org/fwd-stream/-/fwd-stream-1.0.4.tgz", + "integrity": "sha1-7Sgcq+1G/uz5Ie4y3ExQs3KsfPo=", + "dependencies": { + "readable-stream": "~1.0.26-4" + } + }, + "node_modules/fwd-stream/node_modules/isarray": { + "version": "0.0.1", + "resolved": "https://registry.npmjs.org/isarray/-/isarray-0.0.1.tgz", + "integrity": "sha1-ihis/Kmo9Bd+Cav8YDiTmwXR7t8=" + }, + "node_modules/fwd-stream/node_modules/readable-stream": { + "version": "1.0.34", + "resolved": "https://registry.npmjs.org/readable-stream/-/readable-stream-1.0.34.tgz", + "integrity": "sha1-Elgg40vIQtLyqq+v5MKRbuMsFXw=", + "dependencies": { + "core-util-is": "~1.0.0", + "inherits": "~2.0.1", + "isarray": "0.0.1", + "string_decoder": "~0.10.x" + } + }, + "node_modules/fwd-stream/node_modules/string_decoder": { + "version": "0.10.31", + "resolved": "https://registry.npmjs.org/string_decoder/-/string_decoder-0.10.31.tgz", + "integrity": "sha1-YuIDvEF2bGwoyfyEMB2rHFMQ+pQ=" + }, "node_modules/gauge": { "version": "1.2.7", "resolved": "https://registry.npmjs.org/gauge/-/gauge-1.2.7.tgz", @@ -6072,6 +6188,11 @@ "postcss": "^8.1.0" } }, + "node_modules/idb-wrapper": { + "version": "1.7.2", + "resolved": "https://registry.npmjs.org/idb-wrapper/-/idb-wrapper-1.7.2.tgz", + "integrity": "sha512-zfNREywMuf0NzDo9mVsL0yegjsirJxHpKHvWcyRozIqQy89g0a3U+oBPOCN4cc0oCiOuYgZHimzaW/R46G1Mpg==" + }, "node_modules/ieee754": { "version": "1.2.1", "resolved": "https://registry.npmjs.org/ieee754/-/ieee754-1.2.1.tgz", @@ -6223,6 +6344,11 @@ "node": ">=0.8.19" } }, + "node_modules/indexof": { + "version": "0.0.1", + "resolved": "https://registry.npmjs.org/indexof/-/indexof-0.0.1.tgz", + "integrity": "sha1-gtwzbSMrkGIXnQWrMpOmYFn9Q10=" + }, "node_modules/inflight": { "version": "1.0.6", "resolved": "https://registry.npmjs.org/inflight/-/inflight-1.0.6.tgz", @@ -6341,6 +6467,11 @@ "node": ">=0.8.0" } }, + "node_modules/int53": { + "version": "0.2.4", + "resolved": "https://registry.npmjs.org/int53/-/int53-0.2.4.tgz", + "integrity": "sha1-XtjXqtbFxlZ8rmmqf/xKEJ7oD4Y=" + }, "node_modules/internal-slot": { "version": "1.0.3", "resolved": "https://registry.npmjs.org/internal-slot/-/internal-slot-1.0.3.tgz", @@ -6371,6 +6502,14 @@ "loose-envify": "^1.0.0" } }, + "node_modules/is": { + "version": "0.2.7", + "resolved": "https://registry.npmjs.org/is/-/is-0.2.7.tgz", + "integrity": "sha1-OzSixI81mXLzUEKEkZOucmS2NWI=", + "engines": { + "node": "*" + } + }, "node_modules/is-arguments": { "version": "1.1.1", "resolved": "https://registry.npmjs.org/is-arguments/-/is-arguments-1.1.1.tgz", @@ -6560,6 +6699,11 @@ "url": "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/sponsors/ljharb" } }, + "node_modules/is-object": { + "version": "0.1.2", + "resolved": "https://registry.npmjs.org/is-object/-/is-object-0.1.2.tgz", + "integrity": "sha1-AO+8CIFsM8/ErIJR0TLhDcZQmNc=" + }, "node_modules/is-plain-obj": { "version": "2.1.0", "resolved": "https://registry.npmjs.org/is-plain-obj/-/is-plain-obj-2.1.0.tgz", @@ -6693,8 +6837,7 @@ "node_modules/isarray": { "version": "1.0.0", "resolved": "https://registry.npmjs.org/isarray/-/isarray-1.0.0.tgz", - "integrity": "sha1-u5NdSFgsuhaMBoNJV6VKPgcSTxE=", - "dev": true + "integrity": "sha1-u5NdSFgsuhaMBoNJV6VKPgcSTxE=" }, "node_modules/isbinaryfile": { "version": "4.0.8", @@ -6708,6 +6851,11 @@ "url": "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/sponsors/gjtorikian/" } }, + "node_modules/isbuffer": { + "version": "0.0.0", + "resolved": "https://registry.npmjs.org/isbuffer/-/isbuffer-0.0.0.tgz", + "integrity": "sha1-OMFG2d9Si4v5sHAcPUPPEt8/w5s=" + }, "node_modules/isexe": { "version": "2.0.0", "resolved": "https://registry.npmjs.org/isexe/-/isexe-2.0.0.tgz", @@ -6836,6 +6984,9 @@ "resolved": "https://registry.npmjs.org/jsonfile/-/jsonfile-4.0.0.tgz", "integrity": "sha1-h3Gq4HmbZAdrdmQPygWPnBDjPss=", "dev": true, + "dependencies": { + "graceful-fs": "^4.1.6" + }, "optionalDependencies": { "graceful-fs": "^4.1.6" } @@ -6965,6 +7116,209 @@ "resolved": "https://registry.npmjs.org/layout-base/-/layout-base-2.0.1.tgz", "integrity": "sha512-dp3s92+uNI1hWIpPGH3jK2kxE2lMjdXdr+DH8ynZHpd6PUlH6x6cbuXnoMmiNumznqaNO31xu9e79F0uuZ0JFg==" }, + "node_modules/level-blobs": { + "version": "0.1.7", + "resolved": "https://registry.npmjs.org/level-blobs/-/level-blobs-0.1.7.tgz", + "integrity": "sha1-mrm5e7mfHtv594o0M+Ie1WOGva8=", + "dependencies": { + "level-peek": "1.0.6", + "once": "^1.3.0", + "readable-stream": "^1.0.26-4" + } + }, + "node_modules/level-blobs/node_modules/isarray": { + "version": "0.0.1", + "resolved": "https://registry.npmjs.org/isarray/-/isarray-0.0.1.tgz", + "integrity": "sha1-ihis/Kmo9Bd+Cav8YDiTmwXR7t8=" + }, + "node_modules/level-blobs/node_modules/readable-stream": { + "version": "1.1.14", + "resolved": "https://registry.npmjs.org/readable-stream/-/readable-stream-1.1.14.tgz", + "integrity": "sha1-fPTFTvZI44EwhMY23SB54WbAgdk=", + "dependencies": { + "core-util-is": "~1.0.0", + "inherits": "~2.0.1", + "isarray": "0.0.1", + "string_decoder": "~0.10.x" + } + }, + "node_modules/level-blobs/node_modules/string_decoder": { + "version": "0.10.31", + "resolved": "https://registry.npmjs.org/string_decoder/-/string_decoder-0.10.31.tgz", + "integrity": "sha1-YuIDvEF2bGwoyfyEMB2rHFMQ+pQ=" + }, + "node_modules/level-filesystem": { + "version": "1.2.0", + "resolved": "https://registry.npmjs.org/level-filesystem/-/level-filesystem-1.2.0.tgz", + "integrity": "sha1-oArKmRnEpN+v3KaoEI0iWq3/Y7M=", + "dependencies": { + "concat-stream": "^1.4.4", + "errno": "^0.1.1", + "fwd-stream": "^1.0.4", + "level-blobs": "^0.1.7", + "level-peek": "^1.0.6", + "level-sublevel": "^5.2.0", + "octal": "^1.0.0", + "once": "^1.3.0", + "xtend": "^2.2.0" + } + }, + "node_modules/level-filesystem/node_modules/xtend": { + "version": "2.2.0", + "resolved": "https://registry.npmjs.org/xtend/-/xtend-2.2.0.tgz", + "integrity": "sha1-7vax8ZjByN6vrYsXZaBNrUoBxak=", + "engines": { + "node": ">=0.4" + } + }, + "node_modules/level-fix-range": { + "version": "1.0.2", + "resolved": "https://registry.npmjs.org/level-fix-range/-/level-fix-range-1.0.2.tgz", + "integrity": "sha1-vxW5Fa422EcMgh6IPd95zRZCCCg=" + }, + "node_modules/level-hooks": { + "version": "4.5.0", + "resolved": "https://registry.npmjs.org/level-hooks/-/level-hooks-4.5.0.tgz", + "integrity": "sha1-G5rmGSKTDzMF0aYfxNg8gQLA3ZM=", + "dependencies": { + "string-range": "~1.2" + } + }, + "node_modules/level-js": { + "version": "2.2.4", + "resolved": "https://registry.npmjs.org/level-js/-/level-js-2.2.4.tgz", + "integrity": "sha1-vAVfQYBjXUSJtWHJSG+jcOjBFpc=", + "dependencies": { + "abstract-leveldown": "~0.12.0", + "idb-wrapper": "^1.5.0", + "isbuffer": "~0.0.0", + "ltgt": "^2.1.2", + "typedarray-to-buffer": "~1.0.0", + "xtend": "~2.1.2" + } + }, + "node_modules/level-js/node_modules/object-keys": { + "version": "0.4.0", + "resolved": "https://registry.npmjs.org/object-keys/-/object-keys-0.4.0.tgz", + "integrity": "sha1-KKaq50KN0sOpLz2V8hM13SBOAzY=" + }, + "node_modules/level-js/node_modules/xtend": { + "version": "2.1.2", + "resolved": "https://registry.npmjs.org/xtend/-/xtend-2.1.2.tgz", + "integrity": "sha1-bv7MKk2tjmlixJAbM3znuoe10os=", + "dependencies": { + "object-keys": "~0.4.0" + }, + "engines": { + "node": ">=0.4" + } + }, + "node_modules/level-peek": { + "version": "1.0.6", + "resolved": "https://registry.npmjs.org/level-peek/-/level-peek-1.0.6.tgz", + "integrity": "sha1-vsUccqgu5GTTNkNMfIdsP8vM538=", + "dependencies": { + "level-fix-range": "~1.0.2" + } + }, + "node_modules/level-sublevel": { + "version": "5.2.3", + "resolved": "https://registry.npmjs.org/level-sublevel/-/level-sublevel-5.2.3.tgz", + "integrity": "sha1-dEwSxy0ucr543eO5tc2E1iGRQTo=", + "dependencies": { + "level-fix-range": "2.0", + "level-hooks": ">=4.4.0 <5", + "string-range": "~1.2.1", + "xtend": "~2.0.4" + } + }, + "node_modules/level-sublevel/node_modules/level-fix-range": { + "version": "2.0.0", + "resolved": "https://registry.npmjs.org/level-fix-range/-/level-fix-range-2.0.0.tgz", + "integrity": "sha1-xBfWIVlEIVGhnZojZ4aPFyTC1Ug=", + "dependencies": { + "clone": "~0.1.9" + } + }, + "node_modules/level-sublevel/node_modules/object-keys": { + "version": "0.2.0", + "resolved": "https://registry.npmjs.org/object-keys/-/object-keys-0.2.0.tgz", + "integrity": "sha1-zd7AKZiwkb5CvxA1rjLknxy26mc=", + "deprecated": "Please update to the latest object-keys", + "dependencies": { + "foreach": "~2.0.1", + "indexof": "~0.0.1", + "is": "~0.2.6" + } + }, + "node_modules/level-sublevel/node_modules/xtend": { + "version": "2.0.6", + "resolved": "https://registry.npmjs.org/xtend/-/xtend-2.0.6.tgz", + "integrity": "sha1-XqZXptukRwacLlnFihE4ywxebO4=", + "dependencies": { + "is-object": "~0.1.2", + "object-keys": "~0.2.0" + }, + "engines": { + "node": ">=0.4" + } + }, + "node_modules/levelup": { + "version": "0.18.6", + "resolved": "https://registry.npmjs.org/levelup/-/levelup-0.18.6.tgz", + "integrity": "sha1-5qAcsIlhbI7MApHCqb0/DETj5es=", + "dependencies": { + "bl": "~0.8.1", + "deferred-leveldown": "~0.2.0", + "errno": "~0.1.1", + "prr": "~0.0.0", + "readable-stream": "~1.0.26", + "semver": "~2.3.1", + "xtend": "~3.0.0" + } + }, + "node_modules/levelup/node_modules/isarray": { + "version": "0.0.1", + "resolved": "https://registry.npmjs.org/isarray/-/isarray-0.0.1.tgz", + "integrity": "sha1-ihis/Kmo9Bd+Cav8YDiTmwXR7t8=" + }, + "node_modules/levelup/node_modules/prr": { + "version": "0.0.0", + "resolved": "https://registry.npmjs.org/prr/-/prr-0.0.0.tgz", + "integrity": "sha1-GoS4WQgyVQFBGFPQCB7j+obikmo=" + }, + "node_modules/levelup/node_modules/readable-stream": { + "version": "1.0.34", + "resolved": "https://registry.npmjs.org/readable-stream/-/readable-stream-1.0.34.tgz", + "integrity": "sha1-Elgg40vIQtLyqq+v5MKRbuMsFXw=", + "dependencies": { + "core-util-is": "~1.0.0", + "inherits": "~2.0.1", + "isarray": "0.0.1", + "string_decoder": "~0.10.x" + } + }, + "node_modules/levelup/node_modules/semver": { + "version": "2.3.2", + "resolved": "https://registry.npmjs.org/semver/-/semver-2.3.2.tgz", + "integrity": "sha1-uYSPJdbPNjMwc+ye+IVtQvEjPlI=", + "bin": { + "semver": "bin/semver" + } + }, + "node_modules/levelup/node_modules/string_decoder": { + "version": "0.10.31", + "resolved": "https://registry.npmjs.org/string_decoder/-/string_decoder-0.10.31.tgz", + "integrity": "sha1-YuIDvEF2bGwoyfyEMB2rHFMQ+pQ=" + }, + "node_modules/levelup/node_modules/xtend": { + "version": "3.0.0", + "resolved": "https://registry.npmjs.org/xtend/-/xtend-3.0.0.tgz", + "integrity": "sha1-XM50B7r2Qsunvs2laBEcST9ZZlo=", + "engines": { + "node": ">=0.4" + } + }, "node_modules/levn": { "version": "0.4.1", "resolved": "https://registry.npmjs.org/levn/-/levn-0.4.1.tgz", @@ -7256,6 +7610,11 @@ "node": ">=10" } }, + "node_modules/ltgt": { + "version": "2.2.1", + "resolved": "https://registry.npmjs.org/ltgt/-/ltgt-2.2.1.tgz", + "integrity": "sha1-81ypHEk/e3PaDgdJUwTxezH4fuU=" + }, "node_modules/make-dir": { "version": "2.1.0", "resolved": "https://registry.npmjs.org/make-dir/-/make-dir-2.1.0.tgz", @@ -7688,6 +8047,11 @@ "integrity": "sha512-1nh45deeb5olNY7eX82BkPO7SSxR5SSYJiPTrTdFUVYwAl8CKMA5N9PjTYkHiRjisVcxcQ1HXdLhx2qxxJzLNQ==", "dev": true }, + "node_modules/node-int64": { + "version": "0.3.3", + "resolved": "https://registry.npmjs.org/node-int64/-/node-int64-0.3.3.tgz", + "integrity": "sha1-LW5rLs5d6FiLQ9iNG8QbJs0fqE0=" + }, "node_modules/node-polyfill-webpack-plugin": { "version": "1.1.4", "resolved": "https://registry.npmjs.org/node-polyfill-webpack-plugin/-/node-polyfill-webpack-plugin-1.1.4.tgz", @@ -8027,6 +8391,11 @@ "url": "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/sponsors/ljharb" } }, + "node_modules/octal": { + "version": "1.0.0", + "resolved": "https://registry.npmjs.org/octal/-/octal-1.0.0.tgz", + "integrity": "sha1-Y+cWKmjvvrniE1iNWOmJ0eXEUws=" + }, "node_modules/on-finished": { "version": "2.3.0", "resolved": "https://registry.npmjs.org/on-finished/-/on-finished-2.3.0.tgz", @@ -8043,7 +8412,6 @@ "version": "1.4.0", "resolved": "https://registry.npmjs.org/once/-/once-1.4.0.tgz", "integrity": "sha1-WDsap3WWHUsROsF9nFC6753Xa9E=", - "dev": true, "dependencies": { "wrappy": "1" } @@ -8147,6 +8515,20 @@ "node": ">=6" } }, + "node_modules/parquetjs-lite": { + "version": "0.8.7", + "resolved": "https://registry.npmjs.org/parquetjs-lite/-/parquetjs-lite-0.8.7.tgz", + "integrity": "sha512-L0tdHzvy0btLJvAFZvjO0+ru+FL8tHPqiVE90KEnMy5jDqv+WmEy9Ii8SjiC+exAOi3RGyPYIdgJvFijYyEahA==", + "dependencies": { + "int53": "^0.2.4", + "node-int64": "^0.3.3", + "snappyjs": "^0.6.0", + "varint": "^5.0.0" + }, + "engines": { + "node": ">=7.6" + } + }, "node_modules/parse-asn1": { "version": "5.1.6", "resolved": "https://registry.npmjs.org/parse-asn1/-/parse-asn1-5.1.6.tgz", @@ -8481,8 +8863,7 @@ "node_modules/process-nextick-args": { "version": "2.0.1", "resolved": "https://registry.npmjs.org/process-nextick-args/-/process-nextick-args-2.0.1.tgz", - "integrity": "sha512-3ouUOpQhtgrbOa17J7+uxOTpITYWaGP7/AhoR3+A+/1e9skrzelGi/dXzEYyvbxubEF6Wn2ypscTKiKJFFn1ag==", - "dev": true + "integrity": "sha512-3ouUOpQhtgrbOa17J7+uxOTpITYWaGP7/AhoR3+A+/1e9skrzelGi/dXzEYyvbxubEF6Wn2ypscTKiKJFFn1ag==" }, "node_modules/progress": { "version": "2.0.3", @@ -8503,6 +8884,11 @@ "react-is": "^16.8.1" } }, + "node_modules/prr": { + "version": "1.0.1", + "resolved": "https://registry.npmjs.org/prr/-/prr-1.0.1.tgz", + "integrity": "sha1-0/wRS6BplaRexok/SEzrHXj19HY=" + }, "node_modules/public-encrypt": { "version": "4.0.3", "resolved": "https://registry.npmjs.org/public-encrypt/-/public-encrypt-4.0.3.tgz", @@ -9325,6 +9711,11 @@ "node": ">=8" } }, + "node_modules/snappyjs": { + "version": "0.6.1", + "resolved": "https://registry.npmjs.org/snappyjs/-/snappyjs-0.6.1.tgz", + "integrity": "sha512-YIK6I2lsH072UE0aOFxxY1dPDCS43I5ktqHpeAsuLNYWkE5pGxRGWfDM4/vSUfNzXjC1Ivzt3qx31PCLmc9yqg==" + }, "node_modules/socket.io": { "version": "4.3.1", "resolved": "https://registry.npmjs.org/socket.io/-/socket.io-4.3.1.tgz", @@ -9540,6 +9931,11 @@ } ] }, + "node_modules/string-range": { + "version": "1.2.2", + "resolved": "https://registry.npmjs.org/string-range/-/string-range-1.2.2.tgz", + "integrity": "sha1-qJPtNH5yKZvIO++78qaSqNI51d0=" + }, "node_modules/string-width": { "version": "2.1.1", "resolved": "https://registry.npmjs.org/string-width/-/string-width-2.1.1.tgz", @@ -10239,8 +10635,12 @@ "node_modules/typedarray": { "version": "0.0.6", "resolved": "https://registry.npmjs.org/typedarray/-/typedarray-0.0.6.tgz", - "integrity": "sha1-hnrHTjhkGHsdPUfZlqeOxciDB3c=", - "dev": true + "integrity": "sha1-hnrHTjhkGHsdPUfZlqeOxciDB3c=" + }, + "node_modules/typedarray-to-buffer": { + "version": "1.0.4", + "resolved": "https://registry.npmjs.org/typedarray-to-buffer/-/typedarray-to-buffer-1.0.4.tgz", + "integrity": "sha1-m7i6DoQfs/TPH+fCRenz+opf6Zw=" }, "node_modules/typescript": { "version": "4.4.4", @@ -10432,6 +10832,11 @@ "spdx-expression-parse": "^3.0.0" } }, + "node_modules/varint": { + "version": "5.0.2", + "resolved": "https://registry.npmjs.org/varint/-/varint-5.0.2.tgz", + "integrity": "sha512-lKxKYG6H03yCZUpAGOPOsMcGxd1RHCu1iKvEHYDPmTyq2HueGhD73ssNBqqQWfvYs04G9iUFRvmAVLW20Jw6ow==" + }, "node_modules/vary": { "version": "1.1.2", "resolved": "https://registry.npmjs.org/vary/-/vary-1.1.2.tgz", @@ -10815,8 +11220,7 @@ "node_modules/wrappy": { "version": "1.0.2", "resolved": "https://registry.npmjs.org/wrappy/-/wrappy-1.0.2.tgz", - "integrity": "sha1-tSQ9jz7BqjXxNkYFvA0QNuMKtp8=", - "dev": true + "integrity": "sha1-tSQ9jz7BqjXxNkYFvA0QNuMKtp8=" }, "node_modules/ws": { "version": "8.2.3", @@ -12812,6 +13216,21 @@ "resolved": "https://registry.npmjs.org/@xtuc/long/-/long-4.2.2.tgz", "integrity": "sha512-NuHqBY1PB/D8xU6s/thBgOAiAP7HOYDQ32+BFZILJ8ivkUkAHQnWfn6WhL79Owj1qmUnoN/YPhktdIoucipkAQ==" }, + "abstract-leveldown": { + "version": "0.12.4", + "resolved": "https://registry.npmjs.org/abstract-leveldown/-/abstract-leveldown-0.12.4.tgz", + "integrity": "sha1-KeGOYy5g5OIh1YECR4UqY9ey5BA=", + "requires": { + "xtend": "~3.0.0" + }, + "dependencies": { + "xtend": { + "version": "3.0.0", + "resolved": "https://registry.npmjs.org/xtend/-/xtend-3.0.0.tgz", + "integrity": "sha1-XM50B7r2Qsunvs2laBEcST9ZZlo=" + } + } + }, "accepts": { "version": "1.3.7", "resolved": "https://registry.npmjs.org/accepts/-/accepts-1.3.7.tgz", @@ -13179,6 +13598,37 @@ "integrity": "sha512-jDctJ/IVQbZoJykoeHbhXpOlNBqGNcwXJKJog42E5HDPUwQTSdjCHdihjj0DlnheQ7blbT6dHOafNAiS8ooQKA==", "dev": true }, + "bl": { + "version": "0.8.2", + "resolved": "https://registry.npmjs.org/bl/-/bl-0.8.2.tgz", + "integrity": "sha1-yba8oI0bwuoA/Ir7Txpf0eHGbk4=", + "requires": { + "readable-stream": "~1.0.26" + }, + "dependencies": { + "isarray": { + "version": "0.0.1", + "resolved": "https://registry.npmjs.org/isarray/-/isarray-0.0.1.tgz", + "integrity": "sha1-ihis/Kmo9Bd+Cav8YDiTmwXR7t8=" + }, + "readable-stream": { + "version": "1.0.34", + "resolved": "https://registry.npmjs.org/readable-stream/-/readable-stream-1.0.34.tgz", + "integrity": "sha1-Elgg40vIQtLyqq+v5MKRbuMsFXw=", + "requires": { + "core-util-is": "~1.0.0", + "inherits": "~2.0.1", + "isarray": "0.0.1", + "string_decoder": "~0.10.x" + } + }, + "string_decoder": { + "version": "0.10.31", + "resolved": "https://registry.npmjs.org/string_decoder/-/string_decoder-0.10.31.tgz", + "integrity": "sha1-YuIDvEF2bGwoyfyEMB2rHFMQ+pQ=" + } + } + }, "bluebird": { "version": "3.7.2", "resolved": "https://registry.npmjs.org/bluebird/-/bluebird-3.7.2.tgz", @@ -13295,6 +13745,16 @@ "safe-buffer": "^5.1.2" } }, + "browserify-fs": { + "version": "1.0.0", + "resolved": "https://registry.npmjs.org/browserify-fs/-/browserify-fs-1.0.0.tgz", + "integrity": "sha1-8HWqinKdTRcW0GZiDjhvzBMRqW8=", + "requires": { + "level-filesystem": "^1.0.1", + "level-js": "^2.1.3", + "levelup": "^0.18.2" + } + }, "browserify-rsa": { "version": "4.1.0", "resolved": "https://registry.npmjs.org/browserify-rsa/-/browserify-rsa-4.1.0.tgz", @@ -13537,6 +13997,11 @@ } } }, + "clone": { + "version": "0.1.19", + "resolved": "https://registry.npmjs.org/clone/-/clone-0.1.19.tgz", + "integrity": "sha1-YT+2hjmyaklKxTJT4Vsaa9iK2oU=" + }, "clone-deep": { "version": "4.0.1", "resolved": "https://registry.npmjs.org/clone-deep/-/clone-deep-4.0.1.tgz", @@ -13636,7 +14101,6 @@ "version": "1.6.2", "resolved": "https://registry.npmjs.org/concat-stream/-/concat-stream-1.6.2.tgz", "integrity": "sha512-27HBghJxjiZtIk3Ycvn/4kbJk/1uZuJFfuPEns6LaEvpvG1f0hTea8lilrouyo9mVc2GWdcEZ8OLoGmSADlrCw==", - "dev": true, "requires": { "buffer-from": "^1.0.0", "inherits": "^2.0.3", @@ -13648,7 +14112,6 @@ "version": "2.3.7", "resolved": "https://registry.npmjs.org/readable-stream/-/readable-stream-2.3.7.tgz", "integrity": "sha512-Ebho8K4jIbHAxnuxi7o42OrZgF/ZTNcsZj6nRKyUmkhLFq8CHItp/fy6hQZuZmP/n3yZ9VBUbp4zz/mX8hmYPw==", - "dev": true, "requires": { "core-util-is": "~1.0.0", "inherits": "~2.0.3", @@ -13663,7 +14126,6 @@ "version": "1.1.1", "resolved": "https://registry.npmjs.org/string_decoder/-/string_decoder-1.1.1.tgz", "integrity": "sha512-n/ShnvDi6FHbbVfviro+WojiFzv+s8MPMHBczVePfUpDJLwoLT0ht1l4YwBCbi8pJAveEEdnkHyPyTP/mzRfwg==", - "dev": true, "requires": { "safe-buffer": "~5.1.0" } @@ -13755,8 +14217,7 @@ "core-util-is": { "version": "1.0.3", "resolved": "https://registry.npmjs.org/core-util-is/-/core-util-is-1.0.3.tgz", - "integrity": "sha512-ZQBvi1DcpJ4GDqanjucZ2Hj3wEO5pZDS89BWbkcrvdxksJorwUDDZamX9ldFkp9aw2lmBDLgkObEA4DWNJ9FYQ==", - "dev": true + "integrity": "sha512-ZQBvi1DcpJ4GDqanjucZ2Hj3wEO5pZDS89BWbkcrvdxksJorwUDDZamX9ldFkp9aw2lmBDLgkObEA4DWNJ9FYQ==" }, "cors": { "version": "2.8.5", @@ -14073,6 +14534,14 @@ "integrity": "sha512-FJ3UgI4gIl+PHZm53knsuSFpE+nESMr7M4v9QcgB7S63Kj/6WqMiFQJpBBYz1Pt+66bZpP3Q7Lye0Oo9MPKEdg==", "dev": true }, + "deferred-leveldown": { + "version": "0.2.0", + "resolved": "https://registry.npmjs.org/deferred-leveldown/-/deferred-leveldown-0.2.0.tgz", + "integrity": "sha1-LO8fER4cV4cNi7uK8mUOWHzS9bQ=", + "requires": { + "abstract-leveldown": "~0.12.1" + } + }, "define-properties": { "version": "1.1.3", "resolved": "https://registry.npmjs.org/define-properties/-/define-properties-1.1.3.tgz", @@ -14318,6 +14787,14 @@ "integrity": "sha512-/o+BXHmB7ocbHEAs6F2EnG0ogybVVUdkRunTT2glZU9XAaGmhqskrvKwqXuDfNjEO0LZKWdejEEpnq8aM0tOaw==", "dev": true }, + "errno": { + "version": "0.1.8", + "resolved": "https://registry.npmjs.org/errno/-/errno-0.1.8.tgz", + "integrity": "sha512-dJ6oBr5SQ1VSd9qkk7ByRgb/1SH4JZjCHSW/mr63/QcXO9zLVxvJ6Oy13nio03rxpSnVDDjFor75SjVeZWPW/A==", + "requires": { + "prr": "~1.0.1" + } + }, "error-ex": { "version": "1.3.2", "resolved": "https://registry.npmjs.org/error-ex/-/error-ex-1.3.2.tgz", @@ -15146,6 +15623,37 @@ "integrity": "sha1-GwqzvVU7Kg1jmdKcDj6gslIHgyc=", "dev": true }, + "fwd-stream": { + "version": "1.0.4", + "resolved": "https://registry.npmjs.org/fwd-stream/-/fwd-stream-1.0.4.tgz", + "integrity": "sha1-7Sgcq+1G/uz5Ie4y3ExQs3KsfPo=", + "requires": { + "readable-stream": "~1.0.26-4" + }, + "dependencies": { + "isarray": { + "version": "0.0.1", + "resolved": "https://registry.npmjs.org/isarray/-/isarray-0.0.1.tgz", + "integrity": "sha1-ihis/Kmo9Bd+Cav8YDiTmwXR7t8=" + }, + "readable-stream": { + "version": "1.0.34", + "resolved": "https://registry.npmjs.org/readable-stream/-/readable-stream-1.0.34.tgz", + "integrity": "sha1-Elgg40vIQtLyqq+v5MKRbuMsFXw=", + "requires": { + "core-util-is": "~1.0.0", + "inherits": "~2.0.1", + "isarray": "0.0.1", + "string_decoder": "~0.10.x" + } + }, + "string_decoder": { + "version": "0.10.31", + "resolved": "https://registry.npmjs.org/string_decoder/-/string_decoder-0.10.31.tgz", + "integrity": "sha1-YuIDvEF2bGwoyfyEMB2rHFMQ+pQ=" + } + } + }, "gauge": { "version": "1.2.7", "resolved": "https://registry.npmjs.org/gauge/-/gauge-1.2.7.tgz", @@ -15452,6 +15960,11 @@ "dev": true, "requires": {} }, + "idb-wrapper": { + "version": "1.7.2", + "resolved": "https://registry.npmjs.org/idb-wrapper/-/idb-wrapper-1.7.2.tgz", + "integrity": "sha512-zfNREywMuf0NzDo9mVsL0yegjsirJxHpKHvWcyRozIqQy89g0a3U+oBPOCN4cc0oCiOuYgZHimzaW/R46G1Mpg==" + }, "ieee754": { "version": "1.2.1", "resolved": "https://registry.npmjs.org/ieee754/-/ieee754-1.2.1.tgz", @@ -15549,6 +16062,11 @@ "integrity": "sha1-khi5srkoojixPcT7a21XbyMUU+o=", "dev": true }, + "indexof": { + "version": "0.0.1", + "resolved": "https://registry.npmjs.org/indexof/-/indexof-0.0.1.tgz", + "integrity": "sha1-gtwzbSMrkGIXnQWrMpOmYFn9Q10=" + }, "inflight": { "version": "1.0.6", "resolved": "https://registry.npmjs.org/inflight/-/inflight-1.0.6.tgz", @@ -15648,6 +16166,11 @@ } } }, + "int53": { + "version": "0.2.4", + "resolved": "https://registry.npmjs.org/int53/-/int53-0.2.4.tgz", + "integrity": "sha1-XtjXqtbFxlZ8rmmqf/xKEJ7oD4Y=" + }, "internal-slot": { "version": "1.0.3", "resolved": "https://registry.npmjs.org/internal-slot/-/internal-slot-1.0.3.tgz", @@ -15672,6 +16195,11 @@ "loose-envify": "^1.0.0" } }, + "is": { + "version": "0.2.7", + "resolved": "https://registry.npmjs.org/is/-/is-0.2.7.tgz", + "integrity": "sha1-OzSixI81mXLzUEKEkZOucmS2NWI=" + }, "is-arguments": { "version": "1.1.1", "resolved": "https://registry.npmjs.org/is-arguments/-/is-arguments-1.1.1.tgz", @@ -15792,6 +16320,11 @@ "has-tostringtag": "^1.0.0" } }, + "is-object": { + "version": "0.1.2", + "resolved": "https://registry.npmjs.org/is-object/-/is-object-0.1.2.tgz", + "integrity": "sha1-AO+8CIFsM8/ErIJR0TLhDcZQmNc=" + }, "is-plain-obj": { "version": "2.1.0", "resolved": "https://registry.npmjs.org/is-plain-obj/-/is-plain-obj-2.1.0.tgz", @@ -15877,8 +16410,7 @@ "isarray": { "version": "1.0.0", "resolved": "https://registry.npmjs.org/isarray/-/isarray-1.0.0.tgz", - "integrity": "sha1-u5NdSFgsuhaMBoNJV6VKPgcSTxE=", - "dev": true + "integrity": "sha1-u5NdSFgsuhaMBoNJV6VKPgcSTxE=" }, "isbinaryfile": { "version": "4.0.8", @@ -15886,6 +16418,11 @@ "integrity": "sha512-53h6XFniq77YdW+spoRrebh0mnmTxRPTlcuIArO57lmMdq4uBKFKaeTjnb92oYWrSn/LVL+LT+Hap2tFQj8V+w==", "dev": true }, + "isbuffer": { + "version": "0.0.0", + "resolved": "https://registry.npmjs.org/isbuffer/-/isbuffer-0.0.0.tgz", + "integrity": "sha1-OMFG2d9Si4v5sHAcPUPPEt8/w5s=" + }, "isexe": { "version": "2.0.0", "resolved": "https://registry.npmjs.org/isexe/-/isexe-2.0.0.tgz", @@ -16095,6 +16632,203 @@ "resolved": "https://registry.npmjs.org/layout-base/-/layout-base-2.0.1.tgz", "integrity": "sha512-dp3s92+uNI1hWIpPGH3jK2kxE2lMjdXdr+DH8ynZHpd6PUlH6x6cbuXnoMmiNumznqaNO31xu9e79F0uuZ0JFg==" }, + "level-blobs": { + "version": "0.1.7", + "resolved": "https://registry.npmjs.org/level-blobs/-/level-blobs-0.1.7.tgz", + "integrity": "sha1-mrm5e7mfHtv594o0M+Ie1WOGva8=", + "requires": { + "level-peek": "1.0.6", + "once": "^1.3.0", + "readable-stream": "^1.0.26-4" + }, + "dependencies": { + "isarray": { + "version": "0.0.1", + "resolved": "https://registry.npmjs.org/isarray/-/isarray-0.0.1.tgz", + "integrity": "sha1-ihis/Kmo9Bd+Cav8YDiTmwXR7t8=" + }, + "readable-stream": { + "version": "1.1.14", + "resolved": "https://registry.npmjs.org/readable-stream/-/readable-stream-1.1.14.tgz", + "integrity": "sha1-fPTFTvZI44EwhMY23SB54WbAgdk=", + "requires": { + "core-util-is": "~1.0.0", + "inherits": "~2.0.1", + "isarray": "0.0.1", + "string_decoder": "~0.10.x" + } + }, + "string_decoder": { + "version": "0.10.31", + "resolved": "https://registry.npmjs.org/string_decoder/-/string_decoder-0.10.31.tgz", + "integrity": "sha1-YuIDvEF2bGwoyfyEMB2rHFMQ+pQ=" + } + } + }, + "level-filesystem": { + "version": "1.2.0", + "resolved": "https://registry.npmjs.org/level-filesystem/-/level-filesystem-1.2.0.tgz", + "integrity": "sha1-oArKmRnEpN+v3KaoEI0iWq3/Y7M=", + "requires": { + "concat-stream": "^1.4.4", + "errno": "^0.1.1", + "fwd-stream": "^1.0.4", + "level-blobs": "^0.1.7", + "level-peek": "^1.0.6", + "level-sublevel": "^5.2.0", + "octal": "^1.0.0", + "once": "^1.3.0", + "xtend": "^2.2.0" + }, + "dependencies": { + "xtend": { + "version": "2.2.0", + "resolved": "https://registry.npmjs.org/xtend/-/xtend-2.2.0.tgz", + "integrity": "sha1-7vax8ZjByN6vrYsXZaBNrUoBxak=" + } + } + }, + "level-fix-range": { + "version": "1.0.2", + "resolved": "https://registry.npmjs.org/level-fix-range/-/level-fix-range-1.0.2.tgz", + "integrity": "sha1-vxW5Fa422EcMgh6IPd95zRZCCCg=" + }, + "level-hooks": { + "version": "4.5.0", + "resolved": "https://registry.npmjs.org/level-hooks/-/level-hooks-4.5.0.tgz", + "integrity": "sha1-G5rmGSKTDzMF0aYfxNg8gQLA3ZM=", + "requires": { + "string-range": "~1.2" + } + }, + "level-js": { + "version": "2.2.4", + "resolved": "https://registry.npmjs.org/level-js/-/level-js-2.2.4.tgz", + "integrity": "sha1-vAVfQYBjXUSJtWHJSG+jcOjBFpc=", + "requires": { + "abstract-leveldown": "~0.12.0", + "idb-wrapper": "^1.5.0", + "isbuffer": "~0.0.0", + "ltgt": "^2.1.2", + "typedarray-to-buffer": "~1.0.0", + "xtend": "~2.1.2" + }, + "dependencies": { + "object-keys": { + "version": "0.4.0", + "resolved": "https://registry.npmjs.org/object-keys/-/object-keys-0.4.0.tgz", + "integrity": "sha1-KKaq50KN0sOpLz2V8hM13SBOAzY=" + }, + "xtend": { + "version": "2.1.2", + "resolved": "https://registry.npmjs.org/xtend/-/xtend-2.1.2.tgz", + "integrity": "sha1-bv7MKk2tjmlixJAbM3znuoe10os=", + "requires": { + "object-keys": "~0.4.0" + } + } + } + }, + "level-peek": { + "version": "1.0.6", + "resolved": "https://registry.npmjs.org/level-peek/-/level-peek-1.0.6.tgz", + "integrity": "sha1-vsUccqgu5GTTNkNMfIdsP8vM538=", + "requires": { + "level-fix-range": "~1.0.2" + } + }, + "level-sublevel": { + "version": "5.2.3", + "resolved": "https://registry.npmjs.org/level-sublevel/-/level-sublevel-5.2.3.tgz", + "integrity": "sha1-dEwSxy0ucr543eO5tc2E1iGRQTo=", + "requires": { + "level-fix-range": "2.0", + "level-hooks": ">=4.4.0 <5", + "string-range": "~1.2.1", + "xtend": "~2.0.4" + }, + "dependencies": { + "level-fix-range": { + "version": "2.0.0", + "resolved": "https://registry.npmjs.org/level-fix-range/-/level-fix-range-2.0.0.tgz", + "integrity": "sha1-xBfWIVlEIVGhnZojZ4aPFyTC1Ug=", + "requires": { + "clone": "~0.1.9" + } + }, + "object-keys": { + "version": "0.2.0", + "resolved": "https://registry.npmjs.org/object-keys/-/object-keys-0.2.0.tgz", + "integrity": "sha1-zd7AKZiwkb5CvxA1rjLknxy26mc=", + "requires": { + "foreach": "~2.0.1", + "indexof": "~0.0.1", + "is": "~0.2.6" + } + }, + "xtend": { + "version": "2.0.6", + "resolved": "https://registry.npmjs.org/xtend/-/xtend-2.0.6.tgz", + "integrity": "sha1-XqZXptukRwacLlnFihE4ywxebO4=", + "requires": { + "is-object": "~0.1.2", + "object-keys": "~0.2.0" + } + } + } + }, + "levelup": { + "version": "0.18.6", + "resolved": "https://registry.npmjs.org/levelup/-/levelup-0.18.6.tgz", + "integrity": "sha1-5qAcsIlhbI7MApHCqb0/DETj5es=", + "requires": { + "bl": "~0.8.1", + "deferred-leveldown": "~0.2.0", + "errno": "~0.1.1", + "prr": "~0.0.0", + "readable-stream": "~1.0.26", + "semver": "~2.3.1", + "xtend": "~3.0.0" + }, + "dependencies": { + "isarray": { + "version": "0.0.1", + "resolved": "https://registry.npmjs.org/isarray/-/isarray-0.0.1.tgz", + "integrity": "sha1-ihis/Kmo9Bd+Cav8YDiTmwXR7t8=" + }, + "prr": { + "version": "0.0.0", + "resolved": "https://registry.npmjs.org/prr/-/prr-0.0.0.tgz", + "integrity": "sha1-GoS4WQgyVQFBGFPQCB7j+obikmo=" + }, + "readable-stream": { + "version": "1.0.34", + "resolved": "https://registry.npmjs.org/readable-stream/-/readable-stream-1.0.34.tgz", + "integrity": "sha1-Elgg40vIQtLyqq+v5MKRbuMsFXw=", + "requires": { + "core-util-is": "~1.0.0", + "inherits": "~2.0.1", + "isarray": "0.0.1", + "string_decoder": "~0.10.x" + } + }, + "semver": { + "version": "2.3.2", + "resolved": "https://registry.npmjs.org/semver/-/semver-2.3.2.tgz", + "integrity": "sha1-uYSPJdbPNjMwc+ye+IVtQvEjPlI=" + }, + "string_decoder": { + "version": "0.10.31", + "resolved": "https://registry.npmjs.org/string_decoder/-/string_decoder-0.10.31.tgz", + "integrity": "sha1-YuIDvEF2bGwoyfyEMB2rHFMQ+pQ=" + }, + "xtend": { + "version": "3.0.0", + "resolved": "https://registry.npmjs.org/xtend/-/xtend-3.0.0.tgz", + "integrity": "sha1-XM50B7r2Qsunvs2laBEcST9ZZlo=" + } + } + }, "levn": { "version": "0.4.1", "resolved": "https://registry.npmjs.org/levn/-/levn-0.4.1.tgz", @@ -16335,6 +17069,11 @@ "yallist": "^4.0.0" } }, + "ltgt": { + "version": "2.2.1", + "resolved": "https://registry.npmjs.org/ltgt/-/ltgt-2.2.1.tgz", + "integrity": "sha1-81ypHEk/e3PaDgdJUwTxezH4fuU=" + }, "make-dir": { "version": "2.1.0", "resolved": "https://registry.npmjs.org/make-dir/-/make-dir-2.1.0.tgz", @@ -16660,6 +17399,11 @@ "integrity": "sha512-1nh45deeb5olNY7eX82BkPO7SSxR5SSYJiPTrTdFUVYwAl8CKMA5N9PjTYkHiRjisVcxcQ1HXdLhx2qxxJzLNQ==", "dev": true }, + "node-int64": { + "version": "0.3.3", + "resolved": "https://registry.npmjs.org/node-int64/-/node-int64-0.3.3.tgz", + "integrity": "sha1-LW5rLs5d6FiLQ9iNG8QbJs0fqE0=" + }, "node-polyfill-webpack-plugin": { "version": "1.1.4", "resolved": "https://registry.npmjs.org/node-polyfill-webpack-plugin/-/node-polyfill-webpack-plugin-1.1.4.tgz", @@ -16914,6 +17658,11 @@ "es-abstract": "^1.19.1" } }, + "octal": { + "version": "1.0.0", + "resolved": "https://registry.npmjs.org/octal/-/octal-1.0.0.tgz", + "integrity": "sha1-Y+cWKmjvvrniE1iNWOmJ0eXEUws=" + }, "on-finished": { "version": "2.3.0", "resolved": "https://registry.npmjs.org/on-finished/-/on-finished-2.3.0.tgz", @@ -16927,7 +17676,6 @@ "version": "1.4.0", "resolved": "https://registry.npmjs.org/once/-/once-1.4.0.tgz", "integrity": "sha1-WDsap3WWHUsROsF9nFC6753Xa9E=", - "dev": true, "requires": { "wrappy": "1" } @@ -17007,6 +17755,17 @@ "callsites": "^3.0.0" } }, + "parquetjs-lite": { + "version": "0.8.7", + "resolved": "https://registry.npmjs.org/parquetjs-lite/-/parquetjs-lite-0.8.7.tgz", + "integrity": "sha512-L0tdHzvy0btLJvAFZvjO0+ru+FL8tHPqiVE90KEnMy5jDqv+WmEy9Ii8SjiC+exAOi3RGyPYIdgJvFijYyEahA==", + "requires": { + "int53": "^0.2.4", + "node-int64": "^0.3.3", + "snappyjs": "^0.6.0", + "varint": "^5.0.0" + } + }, "parse-asn1": { "version": "5.1.6", "resolved": "https://registry.npmjs.org/parse-asn1/-/parse-asn1-5.1.6.tgz", @@ -17238,8 +17997,7 @@ "process-nextick-args": { "version": "2.0.1", "resolved": "https://registry.npmjs.org/process-nextick-args/-/process-nextick-args-2.0.1.tgz", - "integrity": "sha512-3ouUOpQhtgrbOa17J7+uxOTpITYWaGP7/AhoR3+A+/1e9skrzelGi/dXzEYyvbxubEF6Wn2ypscTKiKJFFn1ag==", - "dev": true + "integrity": "sha512-3ouUOpQhtgrbOa17J7+uxOTpITYWaGP7/AhoR3+A+/1e9skrzelGi/dXzEYyvbxubEF6Wn2ypscTKiKJFFn1ag==" }, "progress": { "version": "2.0.3", @@ -17257,6 +18015,11 @@ "react-is": "^16.8.1" } }, + "prr": { + "version": "1.0.1", + "resolved": "https://registry.npmjs.org/prr/-/prr-1.0.1.tgz", + "integrity": "sha1-0/wRS6BplaRexok/SEzrHXj19HY=" + }, "public-encrypt": { "version": "4.0.3", "resolved": "https://registry.npmjs.org/public-encrypt/-/public-encrypt-4.0.3.tgz", @@ -17887,6 +18650,11 @@ } } }, + "snappyjs": { + "version": "0.6.1", + "resolved": "https://registry.npmjs.org/snappyjs/-/snappyjs-0.6.1.tgz", + "integrity": "sha512-YIK6I2lsH072UE0aOFxxY1dPDCS43I5ktqHpeAsuLNYWkE5pGxRGWfDM4/vSUfNzXjC1Ivzt3qx31PCLmc9yqg==" + }, "socket.io": { "version": "4.3.1", "resolved": "https://registry.npmjs.org/socket.io/-/socket.io-4.3.1.tgz", @@ -18066,6 +18834,11 @@ } } }, + "string-range": { + "version": "1.2.2", + "resolved": "https://registry.npmjs.org/string-range/-/string-range-1.2.2.tgz", + "integrity": "sha1-qJPtNH5yKZvIO++78qaSqNI51d0=" + }, "string-width": { "version": "2.1.1", "resolved": "https://registry.npmjs.org/string-width/-/string-width-2.1.1.tgz", @@ -18583,8 +19356,12 @@ "typedarray": { "version": "0.0.6", "resolved": "https://registry.npmjs.org/typedarray/-/typedarray-0.0.6.tgz", - "integrity": "sha1-hnrHTjhkGHsdPUfZlqeOxciDB3c=", - "dev": true + "integrity": "sha1-hnrHTjhkGHsdPUfZlqeOxciDB3c=" + }, + "typedarray-to-buffer": { + "version": "1.0.4", + "resolved": "https://registry.npmjs.org/typedarray-to-buffer/-/typedarray-to-buffer-1.0.4.tgz", + "integrity": "sha1-m7i6DoQfs/TPH+fCRenz+opf6Zw=" }, "typescript": { "version": "4.4.4", @@ -18731,6 +19508,11 @@ "spdx-expression-parse": "^3.0.0" } }, + "varint": { + "version": "5.0.2", + "resolved": "https://registry.npmjs.org/varint/-/varint-5.0.2.tgz", + "integrity": "sha512-lKxKYG6H03yCZUpAGOPOsMcGxd1RHCu1iKvEHYDPmTyq2HueGhD73ssNBqqQWfvYs04G9iUFRvmAVLW20Jw6ow==" + }, "vary": { "version": "1.1.2", "resolved": "https://registry.npmjs.org/vary/-/vary-1.1.2.tgz", @@ -19006,8 +19788,7 @@ "wrappy": { "version": "1.0.2", "resolved": "https://registry.npmjs.org/wrappy/-/wrappy-1.0.2.tgz", - "integrity": "sha1-tSQ9jz7BqjXxNkYFvA0QNuMKtp8=", - "dev": true + "integrity": "sha1-tSQ9jz7BqjXxNkYFvA0QNuMKtp8=" }, "ws": { "version": "8.2.3", diff --git a/dash/dash-renderer/package.json b/dash/dash-renderer/package.json index a6010a2604..e16495227b 100644 --- a/dash/dash-renderer/package.json +++ b/dash/dash-renderer/package.json @@ -23,6 +23,9 @@ "dependencies": { "@babel/polyfill": "^7.12.1", "@plotly/dash-component-plugins": "^1.2.0", + "browserify-fs": "^1.0.0", + "browserify-zlib": "^0.2.0", + "buffer": "^6.0.3", "cookie": "^0.4.1", "cytoscape": "^3.19.1", "cytoscape-dagre": "^2.3.2", @@ -30,6 +33,8 @@ "dependency-graph": "^0.11.0", "fast-isnumeric": "^1.1.4", "node-polyfill-webpack-plugin": "^1.1.4", + "parquetjs-lite": "^0.8.7", + "path-browserify": "^1.0.1", "prop-types": "15.7.2", "ramda": "^0.27.1", "react": "16.14.0", @@ -40,7 +45,8 @@ "redux": "^4.1.1", "redux-actions": "^2.6.5", "redux-thunk": "^2.3.0", - "regenerator-runtime": "^0.13.9" + "regenerator-runtime": "^0.13.9", + "stream-browserify": "^3.0.0" }, "devDependencies": { "@babel/cli": "^7.15.7", diff --git a/dash/dash-renderer/src/actions/api.js b/dash/dash-renderer/src/actions/api.js index 3f617d5c11..18131e9413 100644 --- a/dash/dash-renderer/src/actions/api.js +++ b/dash/dash-renderer/src/actions/api.js @@ -3,6 +3,7 @@ import {getCSRFHeader, handleAsyncError, addHttpHeaders} from '../actions'; import {urlBase} from './utils'; import {MAX_AUTH_RETRIES} from './constants'; import {JWT_EXPIRED_MESSAGE, STATUS} from '../constants/constants'; +import {deserializeLayout} from '../serializers'; /* eslint-disable-next-line no-console */ const logWarningOnce = once(console.warn); @@ -100,7 +101,8 @@ export default function apiThunk(endpoint, method, store, id, body) { } setConnectionStatus(true); if (contentType && contentType.indexOf('application/json') !== -1) { - return res.json().then(json => { + return res.json().then(async json => { + if (store == 'layoutRequest') await deserializeLayout(json); dispatch({ type: store, payload: { diff --git a/dash/dash-renderer/src/actions/callbacks.ts b/dash/dash-renderer/src/actions/callbacks.ts index bec4167493..49cf7a0c0f 100644 --- a/dash/dash-renderer/src/actions/callbacks.ts +++ b/dash/dash-renderer/src/actions/callbacks.ts @@ -31,6 +31,11 @@ import {urlBase} from './utils'; import {getCSRFHeader} from '.'; import {createAction, Action} from 'redux-actions'; import {addHttpHeaders} from '../actions'; +import { + serializeValue, + SERIALIZER_BOOKKEEPER, + deserializeCbResponse +} from '../serializers'; export const addBlockedCallbacks = createAction( CallbackActionType.AddBlocked @@ -130,7 +135,7 @@ function unwrapIfNotMulti( return [idProps[0], msg]; } -function fillVals( +async function fillVals( paths: any, layout: any, cb: ICallback, @@ -142,13 +147,17 @@ function fillVals( const errors: any[] = []; let emptyMultiValues = 0; - const inputVals = getter(paths).map((inputList: any, i: number) => { + const inputVals = []; + const _paths = getter(paths); + for (let i = 0; i < _paths.length; i++) { + const inputList = _paths[i]; const [inputs, inputError] = unwrapIfNotMulti( paths, inputList.map(({id, property, path: path_}: any) => ({ id, property, - value: (path(path_, layout) as any).props[property] + props: (path(path_, layout) as any).props, + bookkeeper: (path(path_, layout) as any)[SERIALIZER_BOOKKEEPER] })), specs[i], cb.anyVals, @@ -160,8 +169,17 @@ function fillVals( if (inputError) { errors.push(inputError); } - return inputs; - }); + + const {bookkeeper, property, props} = inputs; + inputs.value = await serializeValue( + bookkeeper?.[property] || {}, + props[property], + props + ); + delete inputs.bookkeeper; + delete inputs.props; + inputVals.push(inputs); + } if (errors.length) { if ( @@ -315,98 +333,105 @@ function handleServerside( const requestTime = Date.now(); const body = JSON.stringify(payload); - return fetch( - `${urlBase(config)}_dash-update-component`, - mergeDeepRight(config.fetch, { - method: 'POST', - headers: getCSRFHeader() as any, - body - }) - ).then( - (res: any) => { - const {status} = res; - - function recordProfile(result: any) { - if (config.ui) { - // Callback profiling - only relevant if we're showing the debug ui - const resources = { - __dash_server: 0, - __dash_client: Date.now() - requestTime, - __dash_upload: body.length, - __dash_download: Number( - res.headers.get('Content-Length') - ) - } as any; - - const timingHeaders = - res.headers.get('Server-Timing') || ''; - - timingHeaders.split(',').forEach((header: any) => { - const name = header.split(';')[0]; - const dur = header.match(/;dur=[0-9.]+/); - - if (dur) { - resources[name] = Number(dur[0].slice(5)); + return new Promise((resolve, reject) => { + fetch( + `${urlBase(config)}_dash-update-component`, + mergeDeepRight(config.fetch, { + method: 'POST', + headers: getCSRFHeader() as any, + body + }) + ).then( + async (res: any) => { + const {status} = res; + + function recordProfile(result: any) { + if (config.ui) { + // Callback profiling - only relevant if we're showing the debug ui + const resources = { + __dash_server: 0, + __dash_client: Date.now() - requestTime, + __dash_upload: body.length, + __dash_download: Number( + res.headers.get('Content-Length') + ) + } as any; + + const timingHeaders = + res.headers.get('Server-Timing') || ''; + + timingHeaders.split(',').forEach((header: any) => { + const name = header.split(';')[0]; + const dur = header.match(/;dur=[0-9.]+/); + + if (dur) { + resources[name] = Number(dur[0].slice(5)); + } + }); + + dispatch( + updateResourceUsage({ + id: payload.output, + usage: resources, + status, + result, + inputs: payload.inputs, + state: payload.state + }) + ); + } + } + + if (status === STATUS.OK) { + return res.json().then(async (data: any) => { + const {multi, response} = data; + if (hooks.request_post) { + hooks.request_post(payload, response); } - }); + let result; + if (multi) { + result = response; + } else { + const {output} = payload; + const id = output.substr( + 0, + output.lastIndexOf('.') + ); + result = {[id]: response.props}; + } + + result = await deserializeCbResponse(result); + recordProfile(result); + resolve(result); + }); + } + if (status === STATUS.PREVENT_UPDATE) { + recordProfile({}); + resolve({}); + } + }, + () => { + // fetch rejection - this means the request didn't return, + // we don't get here from 400/500 errors, only network + // errors or unresponsive servers. + if (config.ui) { dispatch( updateResourceUsage({ id: payload.output, - usage: resources, - status, - result, + status: STATUS.NO_RESPONSE, + result: {}, inputs: payload.inputs, state: payload.state }) ); } - } - - if (status === STATUS.OK) { - return res.json().then((data: any) => { - const {multi, response} = data; - if (hooks.request_post) { - hooks.request_post(payload, response); - } - - let result; - if (multi) { - result = response; - } else { - const {output} = payload; - const id = output.substr(0, output.lastIndexOf('.')); - result = {[id]: response.props}; - } - - recordProfile(result); - return result; - }); - } - if (status === STATUS.PREVENT_UPDATE) { - recordProfile({}); - return {}; - } - throw res; - }, - () => { - // fetch rejection - this means the request didn't return, - // we don't get here from 400/500 errors, only network - // errors or unresponsive servers. - if (config.ui) { - dispatch( - updateResourceUsage({ - id: payload.output, - status: STATUS.NO_RESPONSE, - result: {}, - inputs: payload.inputs, - state: payload.state - }) + reject( + new Error('Callback failed: the server did not respond.') ); } - throw new Error('Callback failed: the server did not respond.'); - } - ); + ); + }); } function inputsToDict(inputs_list: any) { @@ -438,7 +463,7 @@ function inputsToDict(inputs_list: any) { return inputs; } -export function executeCallback( +export async function executeCallback( cb: IPrioritizedCallback, config: any, hooks: any, @@ -446,11 +471,11 @@ export function executeCallback( layout: any, {allOutputs}: any, dispatch: any -): IExecutingCallback { +): Promise { const {output, inputs, state, clientside_function} = cb.callback; try { - const inVals = fillVals(paths, layout, cb, inputs, 'Input', true); + const inVals = await fillVals(paths, layout, cb, inputs, 'Input', true); /* Prevent callback if there's no inputs */ if (inVals === null) { @@ -498,7 +523,7 @@ export function executeCallback( inputs: inVals, changedPropIds: keys(cb.changedPropIds), state: cb.callback.state.length - ? fillVals(paths, layout, cb, state, 'State') + ? await fillVals(paths, layout, cb, state, 'State') : undefined }; @@ -530,7 +555,6 @@ export function executeCallback( newConfig, payload ); - if (newHeaders) { dispatch(addHttpHeaders(newHeaders)); } @@ -586,7 +610,7 @@ export function executeCallback( const newCb = { ...cb, - executionPromise: __execute() + executionPromise: await __execute() }; return newCb; diff --git a/dash/dash-renderer/src/observers/prioritizedCallbacks.ts b/dash/dash-renderer/src/observers/prioritizedCallbacks.ts index 7407e24271..94a983ec19 100644 --- a/dash/dash-renderer/src/observers/prioritizedCallbacks.ts +++ b/dash/dash-renderer/src/observers/prioritizedCallbacks.ts @@ -1,4 +1,4 @@ -import {find, flatten, forEach, map, partition, pluck, sort, uniq} from 'ramda'; +import {find, flatten, map, partition, pluck, sort, uniq} from 'ramda'; import {IStoreState} from '../store'; @@ -20,6 +20,7 @@ import isAppReady from '../actions/isAppReady'; import { IBlockedCallback, ICallback, + IExecutingCallback, ILayoutCallbackProperty, IPrioritizedCallback } from '../types/callbacks'; @@ -97,24 +98,26 @@ const observer: IStoreObserverDefinition = { ); if (pickedSyncCallbacks.length) { + const executingCallbacks: IExecutingCallback[] = []; + for (let index = 0; index < pickedSyncCallbacks.length; index++) { + const element = pickedSyncCallbacks[index]; + executingCallbacks.push( + await executeCallback( + element, + config, + hooks, + paths, + layout, + getStash(element, paths), + dispatch + ) + ); + } + dispatch( aggregateCallbacks([ removePrioritizedCallbacks(pickedSyncCallbacks), - addExecutingCallbacks( - map( - cb => - executeCallback( - cb, - config, - hooks, - paths, - layout, - getStash(cb, paths), - dispatch - ), - pickedSyncCallbacks - ) - ) + addExecutingCallbacks(executingCallbacks) ]) ); } @@ -135,8 +138,8 @@ const observer: IStoreObserverDefinition = { addBlockedCallbacks(deferred) ]) ); - - forEach(async cb => { + for (const i in deferred) { + const cb = deferred[i]; await cb.isReady; const { @@ -155,7 +158,7 @@ const observer: IStoreObserverDefinition = { return; } - const executingCallback = executeCallback( + const executingCallback = await executeCallback( cb, config, hooks, @@ -164,14 +167,16 @@ const observer: IStoreObserverDefinition = { cb, dispatch ); - dispatch( aggregateCallbacks([ removeBlockedCallbacks([cb]), addExecutingCallbacks([executingCallback]) ]) ); - }, deferred); + } + // forEach(async cb => { + + // }, deferred); } }, inputs: ['callbacks.prioritized', 'callbacks.completed'] diff --git a/dash/dash-renderer/src/reducers/layout.js b/dash/dash-renderer/src/reducers/layout.js index 6c082f313c..82a986f88b 100644 --- a/dash/dash-renderer/src/reducers/layout.js +++ b/dash/dash-renderer/src/reducers/layout.js @@ -1,6 +1,7 @@ import {append, assocPath, includes, lensPath, mergeRight, view} from 'ramda'; import {getAction} from '../actions/constants'; +import {SERIALIZER_BOOKKEEPER} from '../serializers'; const layout = (state = {}, action) => { if (action.type === getAction('SET_LAYOUT')) { @@ -15,7 +16,17 @@ const layout = (state = {}, action) => { const propPath = append('props', action.payload.itempath); const existingProps = view(lensPath(propPath), state); const mergedProps = mergeRight(existingProps, action.payload.props); - return assocPath(propPath, mergedProps, state); + let newState = state; + + if (action.payload.source === 'response') { + newState = assocPath( + append(SERIALIZER_BOOKKEEPER, action.payload.itempath), + mergedProps[SERIALIZER_BOOKKEEPER], + newState + ); + } + + return assocPath(propPath, mergedProps, newState); } return state; diff --git a/dash/dash-renderer/src/serializers/index.js b/dash/dash-renderer/src/serializers/index.js new file mode 100644 index 0000000000..5144622d28 --- /dev/null +++ b/dash/dash-renderer/src/serializers/index.js @@ -0,0 +1,88 @@ +import {type, prop} from 'ramda'; +import DataFrameSerializer from './pd.dataframe'; + +const PROP_TYPE = '__type'; +const PROP_VALUE = '__value'; +const PROP_ENGINE = '__engine'; +const supportedTypes = { + 'pd.DataFrame': DataFrameSerializer +}; + +export const SERIALIZER_BOOKKEEPER = '__dash_serialized_props'; +export const deserializeCbResponse = async response => { + if (Object.keys(response).length == 1) { + return { + [Object.keys(response)[0]]: ( + await deserializeLayout({props: Object.values(response)[0]}) + ).props + }; + } else { + return (await deserializeLayout({props: response})).props; + } +}; + +export const deserializeLayout = async layout => { + const markedLayout = {...layout, [SERIALIZER_BOOKKEEPER]: {}}; + const { + props, + props: {children} + } = layout; + if (type(children) === 'Array') + for (let index = 0; index < children.length; index++) { + children[index] = await deserializeLayout(children[index]); + } + + if (type(children) === 'Object') + markedLayout.props.children = await deserializeLayout(children); + + for (const [key, value] of Object.entries(props)) { + if (prop(PROP_TYPE, value)) { + await deserializeValue(markedLayout, key, value); + } else { + markedLayout.props[key] = value; + } + } + return markedLayout; +}; + +const deserializeValue = async ( + layout, + key, + {[PROP_TYPE]: type, [PROP_ENGINE]: engine, [PROP_VALUE]: originalValue} +) => { + const [val, missingProps] = (await supportedTypes[type]?.deserialize( + engine, + originalValue + )) || [originalValue, {}]; + layout[SERIALIZER_BOOKKEEPER][key] = { + type, + engine, + autoFilledProps: Object.keys(missingProps) + }; + layout.props[key] = val; + for (const k in missingProps) { + layout.props[k] = missingProps[k]; + } +}; + +export const serializeValue = async ( + {type, engine, autoFilledProps}, + value, + props +) => { + if (!type) return value; + const serializedValue = {}; + serializedValue[PROP_TYPE] = type; + serializedValue[PROP_ENGINE] = engine; + const additionalProps = {}; + autoFilledProps.forEach(p => { + additionalProps[p] = props[p]; + }); + serializedValue[PROP_VALUE] = + (await supportedTypes[type]?.serialize( + engine, + value, + additionalProps + )) || value; + return serializedValue; +}; diff --git a/dash/dash-renderer/src/serializers/pd.dataframe/fastparquet.ts b/dash/dash-renderer/src/serializers/pd.dataframe/fastparquet.ts new file mode 100644 index 0000000000..e60211f43c --- /dev/null +++ b/dash/dash-renderer/src/serializers/pd.dataframe/fastparquet.ts @@ -0,0 +1,82 @@ +const parquet = require('parquetjs-lite'); +import {Buffer} from 'buffer/'; + +function u8array_create(data: string) { + // See https://developer.mozilla.org/en-US/docs/Web/API/btoa for why this is + // necessary. + const byte_chars = atob(data); + const byte_numbers = []; + for (let index = 0; index < byte_chars.length; index++) + byte_numbers.push(byte_chars.charCodeAt(index)); + const byte_array = new Uint8Array(byte_numbers); + return byte_array; +} + +function sanitizeColumnValues(s: any, r: any) { + for (const [k, v] of Object.entries(r)) { + switch (s.fields?.[k]?.primitiveType) { + case 'INT64': + r[k] = Number(v); + break; + default: + break; + } + } + return r; +} + +const fromParquet = async (_parquetFile: string) => { + const records: any[] = []; + const parquetData = Buffer.from(u8array_create(_parquetFile)); + const reader = await parquet.ParquetReader.openBuffer(parquetData); + const schema = reader.getSchema(); + const columns = schema.fieldList.map((field: {name: string}) => ({ + name: field.name, + id: field.name + })); + const cursor = reader.getCursor(); + let record = await cursor.next(); + while (record) { + record = sanitizeColumnValues(schema, record); + records.push(record); + record = await cursor.next(); + } + reader.close(); + const result = [records, {columns, schema}]; + return result; +}; + +const transformPromise = (t: any, d: any, e: any) => + new Promise((resolve, reject) => { + t._transform(d, e, (error: any, data: any) => { + if (!error) resolve(data); + else reject(error); + }); + }); + +const flushPromise = (t: any) => + new Promise((resolve, reject) => + t._flush((error: any, data: string) => { + if (!error) resolve(btoa(data)); + else reject(); + }) + ); + +const toParquet = async (args: any) => { + const [records, additionalProps] = args; + const schema = additionalProps.schema; + const opts = {}; + const transformer = new parquet.ParquetTransformer(schema, opts); + await Promise.allSettled( + records.map((record: any) => + transformPromise(transformer, record, 'utf-8') + ) + ); + const result = await flushPromise(transformer); + return result; +}; + +export default { + serialize: toParquet, + deserialize: fromParquet +}; diff --git a/dash/dash-renderer/src/serializers/pd.dataframe/index.js b/dash/dash-renderer/src/serializers/pd.dataframe/index.js new file mode 100644 index 0000000000..b741e8ae5a --- /dev/null +++ b/dash/dash-renderer/src/serializers/pd.dataframe/index.js @@ -0,0 +1,16 @@ +import FastParquetDataFrameSerializer from './fastparquet'; +import ArrowDataFrameSerializer from './pyarrow'; +import DictDataFrameSerializer from './to_dict'; + +const supportedEngines = { + pyarrow: ArrowDataFrameSerializer, + fastparquet: FastParquetDataFrameSerializer, + to_dict: DictDataFrameSerializer +}; + +export default { + serialize: (engine, ...args) => + supportedEngines[engine]?.serialize(args) || args, + deserialize: (engine, arg) => + supportedEngines[engine]?.deserialize(arg) || [arg] +}; diff --git a/dash/dash-renderer/src/serializers/pd.dataframe/pyarrow.ts b/dash/dash-renderer/src/serializers/pd.dataframe/pyarrow.ts new file mode 100644 index 0000000000..d126e527f0 --- /dev/null +++ b/dash/dash-renderer/src/serializers/pd.dataframe/pyarrow.ts @@ -0,0 +1,41 @@ +// import {Table, Vector} from 'apache-arrow'; +// import {StructRow} from 'apache-arrow/vector/row'; + +/** Data types used by ArrowJS. */ +export type DataType = + | null + | boolean + | number + | string + | Date // datetime + | Int32Array // int + | Uint8Array; // bytes +// | Vector // arrays +// | StructRow; // interval + +export interface ArrowDataframeProto { + data: ArrowTableProto; + height: string; + width: string; +} + +export interface ArrowTableProto { + data: Uint8Array; + index: Uint8Array; + columns: Uint8Array; +} + +export default class ArrowDataFrameSerializer { + static deserialize = (_value: string) => { + // const dataBuffer: Uint8Array = new TextEncoder().encode(atob(value)); + // console.log('dataBuffer = ', dataBuffer); + // const arTable = Table.from(dataBuffer); + // console.log('arTable = ', arTable); + // const records = arTable.toArray(); + // console.log('ArrowTable.toArray() = ', records); + // return records; + return []; + }; + + static serialize = () => {}; +} diff --git a/dash/dash-renderer/src/serializers/pd.dataframe/to_dict.js b/dash/dash-renderer/src/serializers/pd.dataframe/to_dict.js new file mode 100644 index 0000000000..04f89d71ff --- /dev/null +++ b/dash/dash-renderer/src/serializers/pd.dataframe/to_dict.js @@ -0,0 +1,36 @@ +function encodeField(key, value) { + switch (key) { + case 'columns': + return value.map(col => col.name); + default: + return value; + } +} + +function decodeField(key, value) { + switch (key) { + case 'columns': + return value.map(col => ({name: col, id: col})); + default: + return value; + } +} + +export default class DictDataFrameSerializer { + static serialize = args => { + const [value, additionalProps] = args; + const result = {records: value}; + for (const [key, value] of Object.entries(additionalProps)) { + result[key] = encodeField(key, value); + } + return result; + }; + static deserialize = value => { + const {records, ...additionalProps} = value; + const result = [records]; + for (const [key, value] of Object.entries(additionalProps)) { + result.push({[key]: decodeField(key, value)}); + } + return result; + }; +} diff --git a/dash/dash-renderer/webpack.base.config.js b/dash/dash-renderer/webpack.base.config.js index 0952601440..410d041203 100644 --- a/dash/dash-renderer/webpack.base.config.js +++ b/dash/dash-renderer/webpack.base.config.js @@ -5,6 +5,12 @@ const packagejson = require('./package.json'); const dashLibraryName = packagejson.name.replace(/-/g, '_'); const defaults = { + plugins: [ + new webpack.ProvidePlugin({ + Buffer: ['buffer', 'Buffer'], + process: 'process/browser', + }), + ], module: { rules: [ { @@ -30,7 +36,14 @@ const defaults = { ] }, resolve: { - extensions: ['.js', '.ts', '.tsx'] + extensions: ['.js', '.ts', '.tsx'], + alias: { + zlib: require.resolve('browserify-zlib'), + stream: require.resolve('stream-browserify'), + fs: require.resolve('browserify-fs'), + path: require.resolve('path-browserify'), + buffer: require.resolve('buffer'), + }, } }; diff --git a/dash/dash.py b/dash/dash.py index b567e5b9d9..ccf1097d27 100644 --- a/dash/dash.py +++ b/dash/dash.py @@ -61,7 +61,7 @@ map_grouping, grouping_len, ) - +from ._dash_serializer import DashSerializer _flask_compress_version = parse_version(get_distribution("flask-compress").version) @@ -1276,6 +1276,7 @@ def dispatch(self): outputs_list = body.get("outputs") or split_callback_id(output) flask.g.outputs_list = outputs_list # pylint: disable=assigning-non-slot + inputs = DashSerializer.deserialize(inputs) flask.g.input_values = ( # pylint: disable=assigning-non-slot input_values ) = inputs_to_dict(inputs) diff --git a/dash/development/base_component.py b/dash/development/base_component.py index 4cde39805f..9ae8443ddf 100644 --- a/dash/development/base_component.py +++ b/dash/development/base_component.py @@ -5,6 +5,7 @@ import random from .._utils import patch_collections_abc, stringify_id +from .._dash_serializer import DashSerializer MutableSequence = patch_collections_abc("MutableSequence") @@ -176,7 +177,7 @@ def set_random_id(self): def to_plotly_json(self): # Add normal properties props = { - p: getattr(self, p) + p: DashSerializer.serialize_prop(self, p) for p in self._prop_names # pylint: disable=no-member if hasattr(self, p) } diff --git a/requires-install.txt b/requires-install.txt index 476666fbf9..ff25376e22 100644 --- a/requires-install.txt +++ b/requires-install.txt @@ -4,3 +4,6 @@ plotly>=5.0.0 dash_html_components==2.0.0 dash_core_components==2.0.0 dash_table==5.0.0 +pandas +fastparquet +pyarrow \ No newline at end of file diff --git a/tests/unit/dash/test_serializer.py b/tests/unit/dash/test_serializer.py new file mode 100644 index 0000000000..8685ce3f56 --- /dev/null +++ b/tests/unit/dash/test_serializer.py @@ -0,0 +1,16 @@ +from dash import _dash_serializer +import base64 +import io + + +def test_data_frame_serializer_fastparquet(): + import pandas as pd + import numpy as np + + df = pd.DataFrame({"x": np.array([1, 2, 3, 4]), "y": np.array([5, 4, 3, 2])}) + serialized_df = _dash_serializer.DataFrameSerializer._DataFrameSerializer__serialize_using_fastparquet( + df + ) + df_parquet = base64.b64decode(serialized_df.encode("utf-8")) + df2 = pd.read_parquet(io.BytesIO(df_parquet)) + assert all(df == df2)