Training WITH Ray ... 2022-01-28 18:50:15,736 INFO worker.py:843 -- Connecting to existing Ray cluster at address: 10.23.0.2:6379 This cluster consists of 1 nodes in total 16.0 CPU resources in total Timer started ... 2022-01-28 18:50:17 INFO numexpr.utils - Note: NumExpr detected 16 cores but "NUMEXPR_MAX_THREADS" not set, so enforcing safe limit of 8. The configuration for pipeline and policies was chosen automatically. It was written into the config file at 'config.yml'. --------- start graph runner (dask:schema_validator pid=None) 2022-01-28 18:50:18.374272: W tensorflow/stream_executor/platform/default/dso_loader.cc:64] Could not load dynamic library 'libcudart.so.11.0'; dlerror: libcudart.so.11.0: cannot open shared object file: No such file or directory (dask:schema_validator pid=None) 2022-01-28 18:50:18.374291: I tensorflow/stream_executor/cuda/cudart_stub.cc:29] Ignore above cudart dlerror if you do not have a GPU set up on your machine. Traceback (most recent call last): File "/home/galaktoza/anaconda3/envs/raysa_rasa/bin/rasa", line 5, in main() File "/home/galaktoza/Documents/code/Raysa-Rasa/rasa/__main__.py", line 119, in main cmdline_arguments.func(cmdline_arguments) File "/home/galaktoza/Documents/code/Raysa-Rasa/rasa/cli/train.py", line 60, in train_parser.set_defaults(func=lambda args: run_training(args, can_exit=True)) File "/home/galaktoza/Documents/code/Raysa-Rasa/rasa/cli/train.py", line 127, in run_training finetuning_epoch_fraction=args.epoch_fraction, File "/home/galaktoza/Documents/code/Raysa-Rasa/rasa/api.py", line 118, in train_dist finetuning_epoch_fraction=finetuning_epoch_fraction, File "/home/galaktoza/Documents/code/Raysa-Rasa/rasa/model_training.py", line 175, in train_dist **(nlu_additional_arguments or {}), File "/home/galaktoza/Documents/code/Raysa-Rasa/rasa/model_training.py", line 241, in _train_graph_dist is_finetuning=is_finetuning, File "/home/galaktoza/Documents/code/Raysa-Rasa/rasa/engine/training/graph_trainer.py", line 174, in train_dist graph_runner.run(inputs={PLACEHOLDER_IMPORTER: importer}) File "/home/galaktoza/Documents/code/Raysa-Rasa/rasa/engine/runner/dask_on_ray.py", line 132, in run dask_result = ray_dask_get(run_graph, run_targets) File "/home/galaktoza/anaconda3/envs/raysa_rasa/lib/python3.7/site-packages/ray/util/dask/scheduler.py", line 127, in ray_dask_get result = ray_get_unpack(object_refs, progress_bar_actor=pb_actor) File "/home/galaktoza/anaconda3/envs/raysa_rasa/lib/python3.7/site-packages/ray/util/dask/scheduler.py", line 423, in ray_get_unpack return get_result(object_refs) File "/home/galaktoza/anaconda3/envs/raysa_rasa/lib/python3.7/site-packages/ray/util/dask/scheduler.py", line 408, in get_result return ray.get(object_refs) (dask:schema_validator pid=None) 2022-01-28 18:50:20,013 ERROR serialization.py:283 -- __init__() missing 2 required positional arguments: 'model_storage' and 'pruned_schema' (dask:schema_validator pid=None) Traceback (most recent call last): File "/home/galaktoza/anaconda3/envs/raysa_rasa/lib/python3.7/site-packages/ray/_private/client_mode_hook.py", line 105, in wrapper (dask:schema_validator pid=None) File "/home/galaktoza/anaconda3/envs/raysa_rasa/lib/python3.7/site-packages/ray/serialization.py", line 281, in deserialize_objects (dask:schema_validator pid=None) obj = self._deserialize_object(data, metadata, object_ref) (dask:schema_validator pid=None) File "/home/galaktoza/anaconda3/envs/raysa_rasa/lib/python3.7/site-packages/ray/serialization.py", line 194, in _deserialize_object return func(*args, **kwargs) (dask:schema_validator pid=None) return self._deserialize_msgpack_data(data, metadata_fields) (dask:schema_validator pid=None) File "/home/galaktoza/anaconda3/envs/raysa_rasa/lib/python3.7/site-packages/ray/serialization.py", line 172, in _deserialize_msgpack_data (dask:schema_validator pid=None) python_objects = self._deserialize_pickle5_data(pickle5_data) File "/home/galaktoza/anaconda3/envs/raysa_rasa/lib/python3.7/site-packages/ray/worker.py", line 1713, in get (dask:schema_validator pid=None) File "/home/galaktoza/anaconda3/envs/raysa_rasa/lib/python3.7/site-packages/ray/serialization.py", line 162, in _deserialize_pickle5_data (dask:schema_validator pid=None) obj = pickle.loads(in_band) (dask:schema_validator pid=None) File "/home/galaktoza/Documents/code/Raysa-Rasa/rasa/engine/training/hooks.py", line 19, in deserializer_training_hook (dask:schema_validator pid=None) result = TrainingHook(a) (dask:schema_validator pid=None) TypeError: __init__() missing 2 required positional arguments: 'model_storage' and 'pruned_schema' (dask:finetuning_validator pid=None) 2022-01-28 18:50:20,017 ERROR serialization.py:283 -- __init__() missing 2 required positional arguments: 'model_storage' and 'pruned_schema' (dask:finetuning_validator pid=None) Traceback (most recent call last): (dask:finetuning_validator pid=None) File "/home/galaktoza/anaconda3/envs/raysa_rasa/lib/python3.7/site-packages/ray/serialization.py", line 281, in deserialize_objects (dask:finetuning_validator pid=None) obj = self._deserialize_object(data, metadata, object_ref) (dask:finetuning_validator pid=None) File "/home/galaktoza/anaconda3/envs/raysa_rasa/lib/python3.7/site-packages/ray/serialization.py", line 194, in _deserialize_object (dask:finetuning_validator pid=None) return self._deserialize_msgpack_data(data, metadata_fields) (dask:finetuning_validator pid=None) File "/home/galaktoza/anaconda3/envs/raysa_rasa/lib/python3.7/site-packages/ray/serialization.py", line 172, in _deserialize_msgpack_data (dask:finetuning_validator pid=None) python_objects = self._deserialize_pickle5_data(pickle5_data) (dask:finetuning_validator pid=None) File "/home/galaktoza/anaconda3/envs/raysa_rasa/lib/python3.7/site-packages/ray/serialization.py", line 162, in _deserialize_pickle5_data raise value.as_instanceof_cause() (dask:finetuning_validator pid=None) obj = pickle.loads(in_band) (dask:finetuning_validator pid=None) File "/home/galaktoza/Documents/code/Raysa-Rasa/rasa/engine/training/hooks.py", line 19, in deserializer_training_hook (dask:finetuning_validator pid=None) result = TrainingHook(a) (dask:finetuning_validator pid=None) TypeError: __init__() missing 2 required positional arguments: 'model_storage' and 'pruned_schema' ray.exceptions.RayTaskError: ray::dask:train_RegexFeaturizer1 (pid=1956995, ip=10.23.0.2) At least one of the input arguments for this task could not be computed: ray.exceptions.RaySystemError: System error: __init__() missing 2 required positional arguments: 'model_storage' and 'pruned_schema' traceback: Traceback (most recent call last): File "/home/galaktoza/anaconda3/envs/raysa_rasa/lib/python3.7/site-packages/ray/serialization.py", line 281, in deserialize_objects obj = self._deserialize_object(data, metadata, object_ref) File "/home/galaktoza/anaconda3/envs/raysa_rasa/lib/python3.7/site-packages/ray/serialization.py", line 194, in _deserialize_object return self._deserialize_msgpack_data(data, metadata_fields) File "/home/galaktoza/anaconda3/envs/raysa_rasa/lib/python3.7/site-packages/ray/serialization.py", line 172, in _deserialize_msgpack_data python_objects = self._deserialize_pickle5_data(pickle5_data) File "/home/galaktoza/anaconda3/envs/raysa_rasa/lib/python3.7/site-packages/ray/serialization.py", line 162, in _deserialize_pickle5_data obj = pickle.loads(in_band) File "/home/galaktoza/Documents/code/Raysa-Rasa/rasa/engine/training/hooks.py", line 19, in deserializer_training_hook result = TrainingHook(a) TypeError: __init__() missing 2 required positional arguments: 'model_storage' and 'pruned_schema'(dask:nlu_training_data_provider pid=None) 2022-01-28 18:50:20,018 ERROR serialization.py:283 -- __init__() missing 2 required positional arguments: 'model_storage' and 'pruned_schema' (dask:nlu_training_data_provider pid=None) Traceback (most recent call last): (dask:nlu_training_data_provider pid=None) File "/home/galaktoza/anaconda3/envs/raysa_rasa/lib/python3.7/site-packages/ray/serialization.py", line 281, in deserialize_objects (dask:nlu_training_data_provider pid=None) obj = self._deserialize_object(data, metadata, object_ref) (dask:nlu_training_data_provider pid=None) File "/home/galaktoza/anaconda3/envs/raysa_rasa/lib/python3.7/site-packages/ray/serialization.py", line 194, in _deserialize_object (dask:nlu_training_data_provider pid=None) return self._deserialize_msgpack_data(data, metadata_fields) (dask:nlu_training_data_provider pid=None) File "/home/galaktoza/anaconda3/envs/raysa_rasa/lib/python3.7/site-packages/ray/serialization.py", line 172, in _deserialize_msgpack_data (dask:nlu_training_data_provider pid=None) python_objects = self._deserialize_pickle5_data(pickle5_data) (dask:nlu_training_data_provider pid=None) File "/home/galaktoza/anaconda3/envs/raysa_rasa/lib/python3.7/site-packages/ray/serialization.py", line 162, in _deserialize_pickle5_data (dask:nlu_training_data_provider pid=None) obj = pickle.loads(in_band) (dask:nlu_training_data_provider pid=None) File "/home/galaktoza/Documents/code/Raysa-Rasa/rasa/engine/training/hooks.py", line 19, in deserializer_training_hook (dask:nlu_training_data_provider pid=None) result = TrainingHook(a) (dask:nlu_training_data_provider pid=None) TypeError: __init__() missing 2 required positional arguments: 'model_storage' and 'pruned_schema' (dask:story_graph_provider pid=None) 2022-01-28 18:50:20,019 ERROR serialization.py:283 -- __init__() missing 2 required positional arguments: 'model_storage' and 'pruned_schema' (dask:story_graph_provider pid=None) Traceback (most recent call last): (dask:story_graph_provider pid=None) File "/home/galaktoza/anaconda3/envs/raysa_rasa/lib/python3.7/site-packages/ray/serialization.py", line 281, in deserialize_objects (dask:story_graph_provider pid=None) obj = self._deserialize_object(data, metadata, object_ref) (dask:story_graph_provider pid=None) File "/home/galaktoza/anaconda3/envs/raysa_rasa/lib/python3.7/site-packages/ray/serialization.py", line 194, in _deserialize_object (dask:story_graph_provider pid=None) return self._deserialize_msgpack_data(data, metadata_fields) (dask:story_graph_provider pid=None) File "/home/galaktoza/anaconda3/envs/raysa_rasa/lib/python3.7/site-packages/ray/serialization.py", line 172, in _deserialize_msgpack_data (dask:story_graph_provider pid=None) python_objects = self._deserialize_pickle5_data(pickle5_data) (dask:story_graph_provider pid=None) File "/home/galaktoza/anaconda3/envs/raysa_rasa/lib/python3.7/site-packages/ray/serialization.py", line 162, in _deserialize_pickle5_data (dask:story_graph_provider pid=None) obj = pickle.loads(in_band) (dask:story_graph_provider pid=None) File "/home/galaktoza/Documents/code/Raysa-Rasa/rasa/engine/training/hooks.py", line 19, in deserializer_training_hook (dask:story_graph_provider pid=None) result = TrainingHook(a) (dask:story_graph_provider pid=None) TypeError: __init__() missing 2 required positional arguments: 'model_storage' and 'pruned_schema' (dask:run_WhitespaceTokenizer0 pid=None) 2022-01-28 18:50:20,020 ERROR serialization.py:283 -- __init__() missing 2 required positional arguments: 'model_storage' and 'pruned_schema' (dask:run_WhitespaceTokenizer0 pid=None) Traceback (most recent call last): (dask:run_WhitespaceTokenizer0 pid=None) File "/home/galaktoza/anaconda3/envs/raysa_rasa/lib/python3.7/site-packages/ray/serialization.py", line 281, in deserialize_objects (dask:run_WhitespaceTokenizer0 pid=None) obj = self._deserialize_object(data, metadata, object_ref) (dask:run_WhitespaceTokenizer0 pid=None) File "/home/galaktoza/anaconda3/envs/raysa_rasa/lib/python3.7/site-packages/ray/serialization.py", line 194, in _deserialize_object (dask:run_WhitespaceTokenizer0 pid=None) return self._deserialize_msgpack_data(data, metadata_fields) (dask:run_WhitespaceTokenizer0 pid=None) File "/home/galaktoza/anaconda3/envs/raysa_rasa/lib/python3.7/site-packages/ray/serialization.py", line 172, in _deserialize_msgpack_data (dask:run_WhitespaceTokenizer0 pid=None) python_objects = self._deserialize_pickle5_data(pickle5_data) (dask:run_WhitespaceTokenizer0 pid=None) File "/home/galaktoza/anaconda3/envs/raysa_rasa/lib/python3.7/site-packages/ray/serialization.py", line 162, in _deserialize_pickle5_data (dask:run_WhitespaceTokenizer0 pid=None) obj = pickle.loads(in_band) (dask:run_WhitespaceTokenizer0 pid=None) File "/home/galaktoza/Documents/code/Raysa-Rasa/rasa/engine/training/hooks.py", line 19, in deserializer_training_hook (dask:run_WhitespaceTokenizer0 pid=None) result = TrainingHook(a) (dask:run_WhitespaceTokenizer0 pid=None) TypeError: __init__() missing 2 required positional arguments: 'model_storage' and 'pruned_schema' (dask:train_RegexFeaturizer1 pid=None) 2022-01-28 18:50:20,022 ERROR serialization.py:283 -- __init__() missing 2 required positional arguments: 'model_storage' and 'pruned_schema' (dask:train_RegexFeaturizer1 pid=None) Traceback (most recent call last): (dask:train_RegexFeaturizer1 pid=None) File "/home/galaktoza/anaconda3/envs/raysa_rasa/lib/python3.7/site-packages/ray/serialization.py", line 281, in deserialize_objects (dask:train_RegexFeaturizer1 pid=None) obj = self._deserialize_object(data, metadata, object_ref) (dask:train_RegexFeaturizer1 pid=None) File "/home/galaktoza/anaconda3/envs/raysa_rasa/lib/python3.7/site-packages/ray/serialization.py", line 194, in _deserialize_object (dask:train_RegexFeaturizer1 pid=None) return self._deserialize_msgpack_data(data, metadata_fields) (dask:train_RegexFeaturizer1 pid=None) File "/home/galaktoza/anaconda3/envs/raysa_rasa/lib/python3.7/site-packages/ray/serialization.py", line 172, in _deserialize_msgpack_data (dask:train_RegexFeaturizer1 pid=None) python_objects = self._deserialize_pickle5_data(pickle5_data) (dask:train_RegexFeaturizer1 pid=None) File "/home/galaktoza/anaconda3/envs/raysa_rasa/lib/python3.7/site-packages/ray/serialization.py", line 162, in _deserialize_pickle5_data (dask:train_RegexFeaturizer1 pid=None) obj = pickle.loads(in_band) (dask:train_RegexFeaturizer1 pid=None) File "/home/galaktoza/Documents/code/Raysa-Rasa/rasa/engine/training/hooks.py", line 19, in deserializer_training_hook (dask:train_RegexFeaturizer1 pid=None) result = TrainingHook(a) (dask:train_RegexFeaturizer1 pid=None) TypeError: __init__() missing 2 required positional arguments: 'model_storage' and 'pruned_schema' (dask:run_RegexFeaturizer1 pid=None) 2022-01-28 18:50:20,023 ERROR serialization.py:283 -- __init__() missing 2 required positional arguments: 'model_storage' and 'pruned_schema' (dask:run_RegexFeaturizer1 pid=None) Traceback (most recent call last): (dask:run_RegexFeaturizer1 pid=None) File "/home/galaktoza/anaconda3/envs/raysa_rasa/lib/python3.7/site-packages/ray/serialization.py", line 281, in deserialize_objects (dask:run_RegexFeaturizer1 pid=None) obj = self._deserialize_object(data, metadata, object_ref) (dask:run_RegexFeaturizer1 pid=None) File "/home/galaktoza/anaconda3/envs/raysa_rasa/lib/python3.7/site-packages/ray/serialization.py", line 194, in _deserialize_object (dask:run_RegexFeaturizer1 pid=None) return self._deserialize_msgpack_data(data, metadata_fields) (dask:run_RegexFeaturizer1 pid=None) File "/home/galaktoza/anaconda3/envs/raysa_rasa/lib/python3.7/site-packages/ray/serialization.py", line 172, in _deserialize_msgpack_data (dask:run_RegexFeaturizer1 pid=None) python_objects = self._deserialize_pickle5_data(pickle5_data) (dask:run_RegexFeaturizer1 pid=None) File "/home/galaktoza/anaconda3/envs/raysa_rasa/lib/python3.7/site-packages/ray/serialization.py", line 162, in _deserialize_pickle5_data (dask:run_RegexFeaturizer1 pid=None) obj = pickle.loads(in_band) (dask:run_RegexFeaturizer1 pid=None) File "/home/galaktoza/Documents/code/Raysa-Rasa/rasa/engine/training/hooks.py", line 19, in deserializer_training_hook (dask:run_RegexFeaturizer1 pid=None) result = TrainingHook(a) (dask:run_RegexFeaturizer1 pid=None) TypeError: __init__() missing 2 required positional arguments: 'model_storage' and 'pruned_schema' (dask:train_LexicalSyntacticFeaturizer2 pid=None) 2022-01-28 18:50:20,025 ERROR serialization.py:283 -- __init__() missing 2 required positional arguments: 'model_storage' and 'pruned_schema' (dask:train_LexicalSyntacticFeaturizer2 pid=None) Traceback (most recent call last): (dask:train_LexicalSyntacticFeaturizer2 pid=None) File "/home/galaktoza/anaconda3/envs/raysa_rasa/lib/python3.7/site-packages/ray/serialization.py", line 281, in deserialize_objects (dask:train_LexicalSyntacticFeaturizer2 pid=None) obj = self._deserialize_object(data, metadata, object_ref) (dask:train_LexicalSyntacticFeaturizer2 pid=None) File "/home/galaktoza/anaconda3/envs/raysa_rasa/lib/python3.7/site-packages/ray/serialization.py", line 194, in _deserialize_object (dask:train_LexicalSyntacticFeaturizer2 pid=None) return self._deserialize_msgpack_data(data, metadata_fields) (dask:train_LexicalSyntacticFeaturizer2 pid=None) File "/home/galaktoza/anaconda3/envs/raysa_rasa/lib/python3.7/site-packages/ray/serialization.py", line 172, in _deserialize_msgpack_data (dask:train_LexicalSyntacticFeaturizer2 pid=None) python_objects = self._deserialize_pickle5_data(pickle5_data) (dask:train_LexicalSyntacticFeaturizer2 pid=None) File "/home/galaktoza/anaconda3/envs/raysa_rasa/lib/python3.7/site-packages/ray/serialization.py", line 162, in _deserialize_pickle5_data (dask:train_LexicalSyntacticFeaturizer2 pid=None) obj = pickle.loads(in_band) (dask:train_LexicalSyntacticFeaturizer2 pid=None) File "/home/galaktoza/Documents/code/Raysa-Rasa/rasa/engine/training/hooks.py", line 19, in deserializer_training_hook (dask:train_LexicalSyntacticFeaturizer2 pid=None) result = TrainingHook(a) (dask:train_LexicalSyntacticFeaturizer2 pid=None) TypeError: __init__() missing 2 required positional arguments: 'model_storage' and 'pruned_schema' (dask:run_LexicalSyntacticFeaturizer2 pid=None) 2022-01-28 18:50:20,027 ERROR serialization.py:283 -- __init__() missing 2 required positional arguments: 'model_storage' and 'pruned_schema' (dask:run_LexicalSyntacticFeaturizer2 pid=None) Traceback (most recent call last): (dask:run_LexicalSyntacticFeaturizer2 pid=None) File "/home/galaktoza/anaconda3/envs/raysa_rasa/lib/python3.7/site-packages/ray/serialization.py", line 281, in deserialize_objects (dask:run_LexicalSyntacticFeaturizer2 pid=None) obj = self._deserialize_object(data, metadata, object_ref) (dask:run_LexicalSyntacticFeaturizer2 pid=None) File "/home/galaktoza/anaconda3/envs/raysa_rasa/lib/python3.7/site-packages/ray/serialization.py", line 194, in _deserialize_object (dask:run_LexicalSyntacticFeaturizer2 pid=None) return self._deserialize_msgpack_data(data, metadata_fields) (dask:run_LexicalSyntacticFeaturizer2 pid=None) File "/home/galaktoza/anaconda3/envs/raysa_rasa/lib/python3.7/site-packages/ray/serialization.py", line 172, in _deserialize_msgpack_data (dask:run_LexicalSyntacticFeaturizer2 pid=None) python_objects = self._deserialize_pickle5_data(pickle5_data) (dask:run_LexicalSyntacticFeaturizer2 pid=None) File "/home/galaktoza/anaconda3/envs/raysa_rasa/lib/python3.7/site-packages/ray/serialization.py", line 162, in _deserialize_pickle5_data (dask:run_LexicalSyntacticFeaturizer2 pid=None) obj = pickle.loads(in_band) (dask:run_LexicalSyntacticFeaturizer2 pid=None) File "/home/galaktoza/Documents/code/Raysa-Rasa/rasa/engine/training/hooks.py", line 19, in deserializer_training_hook (dask:run_LexicalSyntacticFeaturizer2 pid=None) result = TrainingHook(a) (dask:run_LexicalSyntacticFeaturizer2 pid=None) TypeError: __init__() missing 2 required positional arguments: 'model_storage' and 'pruned_schema' (dask:train_CountVectorsFeaturizer3 pid=None) 2022-01-28 18:50:20,030 ERROR serialization.py:283 -- __init__() missing 2 required positional arguments: 'model_storage' and 'pruned_schema' (dask:train_CountVectorsFeaturizer3 pid=None) Traceback (most recent call last): (dask:train_CountVectorsFeaturizer3 pid=None) File "/home/galaktoza/anaconda3/envs/raysa_rasa/lib/python3.7/site-packages/ray/serialization.py", line 281, in deserialize_objects (dask:train_CountVectorsFeaturizer3 pid=None) obj = self._deserialize_object(data, metadata, object_ref) (dask:train_CountVectorsFeaturizer3 pid=None) File "/home/galaktoza/anaconda3/envs/raysa_rasa/lib/python3.7/site-packages/ray/serialization.py", line 194, in _deserialize_object (dask:train_CountVectorsFeaturizer3 pid=None) return self._deserialize_msgpack_data(data, metadata_fields) (dask:train_CountVectorsFeaturizer3 pid=None) File "/home/galaktoza/anaconda3/envs/raysa_rasa/lib/python3.7/site-packages/ray/serialization.py", line 172, in _deserialize_msgpack_data (dask:train_CountVectorsFeaturizer3 pid=None) python_objects = self._deserialize_pickle5_data(pickle5_data) (dask:train_CountVectorsFeaturizer3 pid=None) File "/home/galaktoza/anaconda3/envs/raysa_rasa/lib/python3.7/site-packages/ray/serialization.py", line 162, in _deserialize_pickle5_data (dask:train_CountVectorsFeaturizer3 pid=None) obj = pickle.loads(in_band) (dask:train_CountVectorsFeaturizer3 pid=None) File "/home/galaktoza/Documents/code/Raysa-Rasa/rasa/engine/training/hooks.py", line 19, in deserializer_training_hook (dask:train_CountVectorsFeaturizer3 pid=None) result = TrainingHook(a) (dask:train_CountVectorsFeaturizer3 pid=None) TypeError: __init__() missing 2 required positional arguments: 'model_storage' and 'pruned_schema' (dask:run_CountVectorsFeaturizer3 pid=None) 2022-01-28 18:50:20,032 ERROR serialization.py:283 -- __init__() missing 2 required positional arguments: 'model_storage' and 'pruned_schema' (dask:run_CountVectorsFeaturizer3 pid=None) Traceback (most recent call last): (dask:run_CountVectorsFeaturizer3 pid=None) File "/home/galaktoza/anaconda3/envs/raysa_rasa/lib/python3.7/site-packages/ray/serialization.py", line 281, in deserialize_objects (dask:run_CountVectorsFeaturizer3 pid=None) obj = self._deserialize_object(data, metadata, object_ref) (dask:run_CountVectorsFeaturizer3 pid=None) File "/home/galaktoza/anaconda3/envs/raysa_rasa/lib/python3.7/site-packages/ray/serialization.py", line 194, in _deserialize_object (dask:run_CountVectorsFeaturizer3 pid=None) return self._deserialize_msgpack_data(data, metadata_fields) (dask:run_CountVectorsFeaturizer3 pid=None) File "/home/galaktoza/anaconda3/envs/raysa_rasa/lib/python3.7/site-packages/ray/serialization.py", line 172, in _deserialize_msgpack_data (dask:run_CountVectorsFeaturizer3 pid=None) python_objects = self._deserialize_pickle5_data(pickle5_data) (dask:run_CountVectorsFeaturizer3 pid=None) File "/home/galaktoza/anaconda3/envs/raysa_rasa/lib/python3.7/site-packages/ray/serialization.py", line 162, in _deserialize_pickle5_data (dask:run_CountVectorsFeaturizer3 pid=None) obj = pickle.loads(in_band) (dask:run_CountVectorsFeaturizer3 pid=None) File "/home/galaktoza/Documents/code/Raysa-Rasa/rasa/engine/training/hooks.py", line 19, in deserializer_training_hook (dask:run_CountVectorsFeaturizer3 pid=None) result = TrainingHook(a) (dask:run_CountVectorsFeaturizer3 pid=None) TypeError: __init__() missing 2 required positional arguments: 'model_storage' and 'pruned_schema' (dask:train_CountVectorsFeaturizer4 pid=None) 2022-01-28 18:50:20,034 ERROR serialization.py:283 -- __init__() missing 2 required positional arguments: 'model_storage' and 'pruned_schema' (dask:train_CountVectorsFeaturizer4 pid=None) Traceback (most recent call last): (dask:train_CountVectorsFeaturizer4 pid=None) File "/home/galaktoza/anaconda3/envs/raysa_rasa/lib/python3.7/site-packages/ray/serialization.py", line 281, in deserialize_objects (dask:train_CountVectorsFeaturizer4 pid=None) obj = self._deserialize_object(data, metadata, object_ref) (dask:train_CountVectorsFeaturizer4 pid=None) File "/home/galaktoza/anaconda3/envs/raysa_rasa/lib/python3.7/site-packages/ray/serialization.py", line 194, in _deserialize_object (dask:train_CountVectorsFeaturizer4 pid=None) return self._deserialize_msgpack_data(data, metadata_fields) (dask:train_CountVectorsFeaturizer4 pid=None) File "/home/galaktoza/anaconda3/envs/raysa_rasa/lib/python3.7/site-packages/ray/serialization.py", line 172, in _deserialize_msgpack_data (dask:train_CountVectorsFeaturizer4 pid=None) python_objects = self._deserialize_pickle5_data(pickle5_data) (dask:train_CountVectorsFeaturizer4 pid=None) File "/home/galaktoza/anaconda3/envs/raysa_rasa/lib/python3.7/site-packages/ray/serialization.py", line 162, in _deserialize_pickle5_data (dask:train_CountVectorsFeaturizer4 pid=None) obj = pickle.loads(in_band) (dask:train_CountVectorsFeaturizer4 pid=None) File "/home/galaktoza/Documents/code/Raysa-Rasa/rasa/engine/training/hooks.py", line 19, in deserializer_training_hook (dask:train_CountVectorsFeaturizer4 pid=None) result = TrainingHook(a) (dask:train_CountVectorsFeaturizer4 pid=None) TypeError: __init__() missing 2 required positional arguments: 'model_storage' and 'pruned_schema' (dask:run_CountVectorsFeaturizer4 pid=None) 2022-01-28 18:50:20,036 ERROR serialization.py:283 -- __init__() missing 2 required positional arguments: 'model_storage' and 'pruned_schema' (dask:run_CountVectorsFeaturizer4 pid=None) Traceback (most recent call last): (dask:run_CountVectorsFeaturizer4 pid=None) File "/home/galaktoza/anaconda3/envs/raysa_rasa/lib/python3.7/site-packages/ray/serialization.py", line 281, in deserialize_objects (dask:run_CountVectorsFeaturizer4 pid=None) obj = self._deserialize_object(data, metadata, object_ref) (dask:run_CountVectorsFeaturizer4 pid=None) File "/home/galaktoza/anaconda3/envs/raysa_rasa/lib/python3.7/site-packages/ray/serialization.py", line 194, in _deserialize_object (dask:run_CountVectorsFeaturizer4 pid=None) return self._deserialize_msgpack_data(data, metadata_fields) (dask:run_CountVectorsFeaturizer4 pid=None) File "/home/galaktoza/anaconda3/envs/raysa_rasa/lib/python3.7/site-packages/ray/serialization.py", line 172, in _deserialize_msgpack_data (dask:run_CountVectorsFeaturizer4 pid=None) python_objects = self._deserialize_pickle5_data(pickle5_data) (dask:run_CountVectorsFeaturizer4 pid=None) File "/home/galaktoza/anaconda3/envs/raysa_rasa/lib/python3.7/site-packages/ray/serialization.py", line 162, in _deserialize_pickle5_data (dask:run_CountVectorsFeaturizer4 pid=None) obj = pickle.loads(in_band) (dask:run_CountVectorsFeaturizer4 pid=None) File "/home/galaktoza/Documents/code/Raysa-Rasa/rasa/engine/training/hooks.py", line 19, in deserializer_training_hook (dask:run_CountVectorsFeaturizer4 pid=None) result = TrainingHook(a) (dask:run_CountVectorsFeaturizer4 pid=None) TypeError: __init__() missing 2 required positional arguments: 'model_storage' and 'pruned_schema' (dask:train_EntitySynonymMapper6 pid=None) 2022-01-28 18:50:20,038 ERROR serialization.py:283 -- __init__() missing 2 required positional arguments: 'model_storage' and 'pruned_schema' (dask:train_EntitySynonymMapper6 pid=None) Traceback (most recent call last): (dask:train_EntitySynonymMapper6 pid=None) File "/home/galaktoza/anaconda3/envs/raysa_rasa/lib/python3.7/site-packages/ray/serialization.py", line 281, in deserialize_objects (dask:train_EntitySynonymMapper6 pid=None) obj = self._deserialize_object(data, metadata, object_ref) (dask:train_EntitySynonymMapper6 pid=None) File "/home/galaktoza/anaconda3/envs/raysa_rasa/lib/python3.7/site-packages/ray/serialization.py", line 194, in _deserialize_object (dask:train_EntitySynonymMapper6 pid=None) return self._deserialize_msgpack_data(data, metadata_fields) (dask:train_EntitySynonymMapper6 pid=None) File "/home/galaktoza/anaconda3/envs/raysa_rasa/lib/python3.7/site-packages/ray/serialization.py", line 172, in _deserialize_msgpack_data (dask:train_EntitySynonymMapper6 pid=None) python_objects = self._deserialize_pickle5_data(pickle5_data) (dask:train_EntitySynonymMapper6 pid=None) File "/home/galaktoza/anaconda3/envs/raysa_rasa/lib/python3.7/site-packages/ray/serialization.py", line 162, in _deserialize_pickle5_data (dask:train_EntitySynonymMapper6 pid=None) obj = pickle.loads(in_band) (dask:train_EntitySynonymMapper6 pid=None) File "/home/galaktoza/Documents/code/Raysa-Rasa/rasa/engine/training/hooks.py", line 19, in deserializer_training_hook (dask:train_EntitySynonymMapper6 pid=None) result = TrainingHook(a) (dask:train_EntitySynonymMapper6 pid=None) TypeError: __init__() missing 2 required positional arguments: 'model_storage' and 'pruned_schema' (dask:train_ResponseSelector7 pid=None) 2022-01-28 18:50:20,040 ERROR serialization.py:283 -- __init__() missing 2 required positional arguments: 'model_storage' and 'pruned_schema' (dask:train_ResponseSelector7 pid=None) Traceback (most recent call last): (dask:train_ResponseSelector7 pid=None) File "/home/galaktoza/anaconda3/envs/raysa_rasa/lib/python3.7/site-packages/ray/serialization.py", line 281, in deserialize_objects (dask:train_ResponseSelector7 pid=None) obj = self._deserialize_object(data, metadata, object_ref) (dask:train_ResponseSelector7 pid=None) File "/home/galaktoza/anaconda3/envs/raysa_rasa/lib/python3.7/site-packages/ray/serialization.py", line 194, in _deserialize_object (dask:train_ResponseSelector7 pid=None) return self._deserialize_msgpack_data(data, metadata_fields) (dask:train_ResponseSelector7 pid=None) File "/home/galaktoza/anaconda3/envs/raysa_rasa/lib/python3.7/site-packages/ray/serialization.py", line 172, in _deserialize_msgpack_data (dask:train_ResponseSelector7 pid=None) python_objects = self._deserialize_pickle5_data(pickle5_data) (dask:train_ResponseSelector7 pid=None) File "/home/galaktoza/anaconda3/envs/raysa_rasa/lib/python3.7/site-packages/ray/serialization.py", line 162, in _deserialize_pickle5_data (dask:train_ResponseSelector7 pid=None) obj = pickle.loads(in_band) (dask:train_ResponseSelector7 pid=None) File "/home/galaktoza/Documents/code/Raysa-Rasa/rasa/engine/training/hooks.py", line 19, in deserializer_training_hook (dask:train_ResponseSelector7 pid=None) result = TrainingHook(a) (dask:train_ResponseSelector7 pid=None) TypeError: __init__() missing 2 required positional arguments: 'model_storage' and 'pruned_schema' (dask:train_DIETClassifier5 pid=None) 2022-01-28 18:50:20,041 ERROR serialization.py:283 -- __init__() missing 2 required positional arguments: 'model_storage' and 'pruned_schema' (dask:train_DIETClassifier5 pid=None) Traceback (most recent call last): (dask:train_DIETClassifier5 pid=None) File "/home/galaktoza/anaconda3/envs/raysa_rasa/lib/python3.7/site-packages/ray/serialization.py", line 281, in deserialize_objects (dask:train_DIETClassifier5 pid=None) obj = self._deserialize_object(data, metadata, object_ref) (dask:train_DIETClassifier5 pid=None) File "/home/galaktoza/anaconda3/envs/raysa_rasa/lib/python3.7/site-packages/ray/serialization.py", line 194, in _deserialize_object (dask:train_DIETClassifier5 pid=None) return self._deserialize_msgpack_data(data, metadata_fields) (dask:train_DIETClassifier5 pid=None) File "/home/galaktoza/anaconda3/envs/raysa_rasa/lib/python3.7/site-packages/ray/serialization.py", line 172, in _deserialize_msgpack_data (dask:train_DIETClassifier5 pid=None) python_objects = self._deserialize_pickle5_data(pickle5_data) (dask:train_DIETClassifier5 pid=None) File "/home/galaktoza/anaconda3/envs/raysa_rasa/lib/python3.7/site-packages/ray/serialization.py", line 162, in _deserialize_pickle5_data (dask:train_DIETClassifier5 pid=None) obj = pickle.loads(in_band) (dask:train_DIETClassifier5 pid=None) File "/home/galaktoza/Documents/code/Raysa-Rasa/rasa/engine/training/hooks.py", line 19, in deserializer_training_hook (dask:train_DIETClassifier5 pid=None) result = TrainingHook(a) (dask:train_DIETClassifier5 pid=None) TypeError: __init__() missing 2 required positional arguments: 'model_storage' and 'pruned_schema' (dask:domain_provider pid=None) 2022-01-28 18:50:20.238134: W tensorflow/stream_executor/platform/default/dso_loader.cc:64] Could not load dynamic library 'libcudart.so.11.0'; dlerror: libcudart.so.11.0: cannot open shared object file: No such file or directory (dask:domain_provider pid=None) 2022-01-28 18:50:20.238155: I tensorflow/stream_executor/cuda/cudart_stub.cc:29] Ignore above cudart dlerror if you do not have a GPU set up on your machine.