I am training a Keras model using custom training loops in TensorFlow, where the weights are updated using gradient tape rather than the model.fit() method. As such, the model is not compiled before training.
After exporting the saved_model, I am able to successfully load it for inference:
model = tf.saved_model.load("path/to/saved_model")
pred_fn = model.signatures["serving_default"]
results = pred_fn(tf.constant(examples))
However, when I try loading it with TFMA using run_model_analysis:
eval_shared_model = tfma.default_eval_shared_model("path/to/saved_model", eval_config=eval_config)
eval_results = tfma.run_model_analysis(
eval_shared_model=eval_shared_model,
data_location=test_tfrecords_path,
file_format="tfrecords"
)
I get the following error:
WARNING:tensorflow:No training configuration found in save file, so the model was *not* compiled. Compile it manually.
-----------------------------------------------------
AttributeError Traceback (most recent call last)
<ipython-input-107-19f51f42014a> in <module>
2 eval_shared_model=eval_shared_model,
3 data_location=test_tfrecords_path,
----> 4 file_format="tfrecords"
5 )
~/.pyenv/versions/miniconda3-4.3.30/envs/tensorflow/lib/python3.7/site-packages/tensorflow_model_analysis/api/model_eval_lib.py in run_model_analysis(eval_shared_model, eval_config, data_location, file_format, output_path, extractors, evaluators, writers, pipeline_options, slice_spec, write_config, compute_confidence_intervals, min_slice_size, random_seed_for_testing, schema)
1200 random_seed_for_testing=random_seed_for_testing,
1201 tensor_adapter_config=tensor_adapter_config,
-> 1202 schema=schema))
1203 # pylint: enable=no-value-for-parameter
1204
~/.pyenv/versions/miniconda3-4.3.30/envs/tensorflow/lib/python3.7/site-packages/apache_beam/pvalue.py in __or__(self, ptransform)
138
139 def __or__(self, ptransform):
--> 140 return self.pipeline.apply(ptransform, self)
141
142
~/.pyenv/versions/miniconda3-4.3.30/envs/tensorflow/lib/python3.7/site-packages/apache_beam/pipeline.py in apply(self, transform, pvalueish, label)
575 if isinstance(transform, ptransform._NamedPTransform):
576 return self.apply(
--> 577 transform.transform, pvalueish, label or transform.label)
578
579 if not isinstance(transform, ptransform.PTransform):
~/.pyenv/versions/miniconda3-4.3.30/envs/tensorflow/lib/python3.7/site-packages/apache_beam/pipeline.py in apply(self, transform, pvalueish, label)
585 try:
586 old_label, transform.label = transform.label, label
--> 587 return self.apply(transform, pvalueish)
588 finally:
589 transform.label = old_label
~/.pyenv/versions/miniconda3-4.3.30/envs/tensorflow/lib/python3.7/site-packages/apache_beam/pipeline.py in apply(self, transform, pvalueish, label)
628 transform.type_check_inputs(pvalueish)
629
--> 630 pvalueish_result = self.runner.apply(transform, pvalueish, self._options)
631
632 if type_options is not None and type_options.pipeline_type_check:
~/.pyenv/versions/miniconda3-4.3.30/envs/tensorflow/lib/python3.7/site-packages/apache_beam/runners/runner.py in apply(self, transform, input, options)
196 m = getattr(self, 'apply_%s' % cls.__name__, None)
197 if m:
--> 198 return m(transform, input, options)
199 raise NotImplementedError(
200 'Execution of [%s] not implemented in runner %s.' % (transform, self))
~/.pyenv/versions/miniconda3-4.3.30/envs/tensorflow/lib/python3.7/site-packages/apache_beam/runners/runner.py in apply_PTransform(self, transform, input, options)
226 def apply_PTransform(self, transform, input, options):
227 # The base case of apply is to call the transform's expand.
--> 228 return transform.expand(input)
229
230 def run_transform(self,
~/.pyenv/versions/miniconda3-4.3.30/envs/tensorflow/lib/python3.7/site-packages/apache_beam/transforms/ptransform.py in expand(self, pcoll)
921 # Might not be a function.
922 pass
--> 923 return self._fn(pcoll, *args, **kwargs)
924
925 def default_label(self):
~/.pyenv/versions/miniconda3-4.3.30/envs/tensorflow/lib/python3.7/site-packages/tensorflow_model_analysis/api/model_eval_lib.py in ExtractEvaluateAndWriteResults(examples, eval_shared_model, eval_config, extractors, evaluators, writers, output_path, display_only_data_location, display_only_file_format, slice_spec, write_config, compute_confidence_intervals, min_slice_size, random_seed_for_testing, tensor_adapter_config, schema)
1079 | 'ExtractAndEvaluate' >> ExtractAndEvaluate(
1080 extractors=extractors, evaluators=evaluators)
-> 1081 | 'WriteResults' >> WriteResults(writers=writers))
1082
1083 return beam.pvalue.PDone(examples.pipeline)
~/.pyenv/versions/miniconda3-4.3.30/envs/tensorflow/lib/python3.7/site-packages/apache_beam/pvalue.py in __or__(self, ptransform)
138
139 def __or__(self, ptransform):
--> 140 return self.pipeline.apply(ptransform, self)
141
142
~/.pyenv/versions/miniconda3-4.3.30/envs/tensorflow/lib/python3.7/site-packages/apache_beam/pipeline.py in apply(self, transform, pvalueish, label)
575 if isinstance(transform, ptransform._NamedPTransform):
576 return self.apply(
--> 577 transform.transform, pvalueish, label or transform.label)
578
579 if not isinstance(transform, ptransform.PTransform):
~/.pyenv/versions/miniconda3-4.3.30/envs/tensorflow/lib/python3.7/site-packages/apache_beam/pipeline.py in apply(self, transform, pvalueish, label)
585 try:
586 old_label, transform.label = transform.label, label
--> 587 return self.apply(transform, pvalueish)
588 finally:
589 transform.label = old_label
~/.pyenv/versions/miniconda3-4.3.30/envs/tensorflow/lib/python3.7/site-packages/apache_beam/pipeline.py in apply(self, transform, pvalueish, label)
628 transform.type_check_inputs(pvalueish)
629
--> 630 pvalueish_result = self.runner.apply(transform, pvalueish, self._options)
631
632 if type_options is not None and type_options.pipeline_type_check:
~/.pyenv/versions/miniconda3-4.3.30/envs/tensorflow/lib/python3.7/site-packages/apache_beam/runners/runner.py in apply(self, transform, input, options)
196 m = getattr(self, 'apply_%s' % cls.__name__, None)
197 if m:
--> 198 return m(transform, input, options)
199 raise NotImplementedError(
200 'Execution of [%s] not implemented in runner %s.' % (transform, self))
~/.pyenv/versions/miniconda3-4.3.30/envs/tensorflow/lib/python3.7/site-packages/apache_beam/runners/runner.py in apply_PTransform(self, transform, input, options)
226 def apply_PTransform(self, transform, input, options):
227 # The base case of apply is to call the transform's expand.
--> 228 return transform.expand(input)
229
230 def run_transform(self,
~/.pyenv/versions/miniconda3-4.3.30/envs/tensorflow/lib/python3.7/site-packages/apache_beam/transforms/ptransform.py in expand(self, pcoll)
921 # Might not be a function.
922 pass
--> 923 return self._fn(pcoll, *args, **kwargs)
924
925 def default_label(self):
~/.pyenv/versions/miniconda3-4.3.30/envs/tensorflow/lib/python3.7/site-packages/tensorflow_model_analysis/api/model_eval_lib.py in ExtractAndEvaluate(extracts, extractors, evaluators)
818 for v in evaluators:
819 if v.run_after == x.stage_name:
--> 820 update(evaluation, extracts | v.stage_name >> v.ptransform)
821 for v in evaluators:
822 if v.run_after == extractor.LAST_EXTRACTOR_STAGE_NAME:
~/.pyenv/versions/miniconda3-4.3.30/envs/tensorflow/lib/python3.7/site-packages/apache_beam/pvalue.py in __or__(self, ptransform)
138
139 def __or__(self, ptransform):
--> 140 return self.pipeline.apply(ptransform, self)
141
142
~/.pyenv/versions/miniconda3-4.3.30/envs/tensorflow/lib/python3.7/site-packages/apache_beam/pipeline.py in apply(self, transform, pvalueish, label)
575 if isinstance(transform, ptransform._NamedPTransform):
576 return self.apply(
--> 577 transform.transform, pvalueish, label or transform.label)
578
579 if not isinstance(transform, ptransform.PTransform):
~/.pyenv/versions/miniconda3-4.3.30/envs/tensorflow/lib/python3.7/site-packages/apache_beam/pipeline.py in apply(self, transform, pvalueish, label)
585 try:
586 old_label, transform.label = transform.label, label
--> 587 return self.apply(transform, pvalueish)
588 finally:
589 transform.label = old_label
~/.pyenv/versions/miniconda3-4.3.30/envs/tensorflow/lib/python3.7/site-packages/apache_beam/pipeline.py in apply(self, transform, pvalueish, label)
628 transform.type_check_inputs(pvalueish)
629
--> 630 pvalueish_result = self.runner.apply(transform, pvalueish, self._options)
631
632 if type_options is not None and type_options.pipeline_type_check:
~/.pyenv/versions/miniconda3-4.3.30/envs/tensorflow/lib/python3.7/site-packages/apache_beam/runners/runner.py in apply(self, transform, input, options)
196 m = getattr(self, 'apply_%s' % cls.__name__, None)
197 if m:
--> 198 return m(transform, input, options)
199 raise NotImplementedError(
200 'Execution of [%s] not implemented in runner %s.' % (transform, self))
~/.pyenv/versions/miniconda3-4.3.30/envs/tensorflow/lib/python3.7/site-packages/apache_beam/runners/runner.py in apply_PTransform(self, transform, input, options)
226 def apply_PTransform(self, transform, input, options):
227 # The base case of apply is to call the transform's expand.
--> 228 return transform.expand(input)
229
230 def run_transform(self,
~/.pyenv/versions/miniconda3-4.3.30/envs/tensorflow/lib/python3.7/site-packages/apache_beam/transforms/ptransform.py in expand(self, pcoll)
921 # Might not be a function.
922 pass
--> 923 return self._fn(pcoll, *args, **kwargs)
924
925 def default_label(self):
~/.pyenv/versions/miniconda3-4.3.30/envs/tensorflow/lib/python3.7/site-packages/tensorflow_model_analysis/evaluators/metrics_and_plots_evaluator_v2.py in _EvaluateMetricsAndPlots(extracts, eval_config, eval_shared_models, metrics_key, plots_key, validations_key, schema, random_seed_for_testing)
757 plots_key=plots_key,
758 schema=schema,
--> 759 random_seed_for_testing=random_seed_for_testing))
760
761 for k, v in evaluation.items():
~/.pyenv/versions/miniconda3-4.3.30/envs/tensorflow/lib/python3.7/site-packages/apache_beam/pvalue.py in __or__(self, ptransform)
138
139 def __or__(self, ptransform):
--> 140 return self.pipeline.apply(ptransform, self)
141
142
~/.pyenv/versions/miniconda3-4.3.30/envs/tensorflow/lib/python3.7/site-packages/apache_beam/pipeline.py in apply(self, transform, pvalueish, label)
575 if isinstance(transform, ptransform._NamedPTransform):
576 return self.apply(
--> 577 transform.transform, pvalueish, label or transform.label)
578
579 if not isinstance(transform, ptransform.PTransform):
~/.pyenv/versions/miniconda3-4.3.30/envs/tensorflow/lib/python3.7/site-packages/apache_beam/pipeline.py in apply(self, transform, pvalueish, label)
585 try:
586 old_label, transform.label = transform.label, label
--> 587 return self.apply(transform, pvalueish)
588 finally:
589 transform.label = old_label
~/.pyenv/versions/miniconda3-4.3.30/envs/tensorflow/lib/python3.7/site-packages/apache_beam/pipeline.py in apply(self, transform, pvalueish, label)
628 transform.type_check_inputs(pvalueish)
629
--> 630 pvalueish_result = self.runner.apply(transform, pvalueish, self._options)
631
632 if type_options is not None and type_options.pipeline_type_check:
~/.pyenv/versions/miniconda3-4.3.30/envs/tensorflow/lib/python3.7/site-packages/apache_beam/runners/runner.py in apply(self, transform, input, options)
196 m = getattr(self, 'apply_%s' % cls.__name__, None)
197 if m:
--> 198 return m(transform, input, options)
199 raise NotImplementedError(
200 'Execution of [%s] not implemented in runner %s.' % (transform, self))
~/.pyenv/versions/miniconda3-4.3.30/envs/tensorflow/lib/python3.7/site-packages/apache_beam/runners/runner.py in apply_PTransform(self, transform, input, options)
226 def apply_PTransform(self, transform, input, options):
227 # The base case of apply is to call the transform's expand.
--> 228 return transform.expand(input)
229
230 def run_transform(self,
~/.pyenv/versions/miniconda3-4.3.30/envs/tensorflow/lib/python3.7/site-packages/apache_beam/transforms/ptransform.py in expand(self, pcoll)
921 # Might not be a function.
922 pass
--> 923 return self._fn(pcoll, *args, **kwargs)
924
925 def default_label(self):
~/.pyenv/versions/miniconda3-4.3.30/envs/tensorflow/lib/python3.7/site-packages/tensorflow_model_analysis/evaluators/metrics_and_plots_evaluator_v2.py in _ComputeMetricsAndPlots(extracts, eval_config, metrics_specs, eval_shared_models, metrics_key, plots_key, schema, random_seed_for_testing)
582 if eval_shared_model.model_type == constants.TF_KERAS:
583 keras_specs = keras_util.metrics_specs_from_keras(
--> 584 model_name, eval_shared_model.model_loader)
585 metrics_specs = keras_specs + metrics_specs[:]
586 # TODO(mdreves): Add support for calling keras.evaluate().
~/.pyenv/versions/miniconda3-4.3.30/envs/tensorflow/lib/python3.7/site-packages/tensorflow_model_analysis/evaluators/keras_util.py in metrics_specs_from_keras(model_name, model_loader)
60 # y_true, y_pred as inputs so it can't be calculated via standard inputs so
61 # we remove it.
---> 62 metrics.extend(model.compiled_loss.metrics[1:])
63 metrics.extend(model.compiled_metrics.metrics)
64 metric_names = [m.name for m in metrics]
AttributeError: 'NoneType' object has no attribute 'metrics'
I suspect this might be because I am not compiling the Keras model before exporting it. Does TFMA only support compiled models?
I am using tensorflow==2.3.0 and tensorflow-model-analysis==0.22.1
Yes, your understanding is correct i.e., it is resulting in
errorbecause you are notcompilingand consequently, not adding theMETRICS.It is evident from the statement specified in the Tensorflow Model Analysis Documentation as well, which is mentioned below.