From 1244b92ea1f0561e4e8453ca241d92480be3e333 Mon Sep 17 00:00:00 2001 From: Karan Shukla Date: Wed, 15 Apr 2020 21:47:59 -0700 Subject: [PATCH 1/9] test.py --- test_runner.py => test.py | 34 +++++++++++++++++++++++++++++++++- 1 file changed, 33 insertions(+), 1 deletion(-) rename test_runner.py => test.py (58%) diff --git a/test_runner.py b/test.py similarity index 58% rename from test_runner.py rename to test.py index dad5c691..85265aa0 100644 --- a/test_runner.py +++ b/test.py @@ -1,6 +1,38 @@ +import os import unittest +from nbconvert.preprocessors import ExecutePreprocessor +import nbformat -import notebook_runner + +# TODO(karanshukla): names: _run_notebook and _write_notebook? +def _run_notebook(path): + + # TODO(karanshukla): can I comment this out? + def _write_notebook(notebook): + dirname = os.path.dirname(path) + notebookname, _ = os.path.splitext(os.path.basename(path)) + output_path = os.path.join(dirname, + '{}_all_output.ipynb'.format(notebookname)) + with open(output_path, mode='wt') as f: + nbformat.write(notebook, f) + + with open(path) as f: + notebook = nbformat.read(f, as_version=4) + + proc = ExecutePreprocessor(timeout=None, kernel_name='python3') + proc.allow_errors = True + proc.preprocess(notebook, {'metadata': {'path': '/'}}) + + _write_notebook(notebook) + + errors = [] + for cell in notebook.cells: + if 'outputs' in cell: + for output in cell['outputs']: + if output.output_type == 'error': + errors.append(output) + + return errors class TestNotebook(unittest.TestCase): From 4df9b06755658e79eed2c1463fe3a4d2bb4e8427 Mon Sep 17 00:00:00 2001 From: Karan Shukla Date: Wed, 15 Apr 2020 21:48:34 -0700 Subject: [PATCH 2/9] test.py --- .github/workflows/jupyter.yml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.github/workflows/jupyter.yml b/.github/workflows/jupyter.yml index af8c49f2..301f35c3 100644 --- a/.github/workflows/jupyter.yml +++ b/.github/workflows/jupyter.yml @@ -47,4 +47,4 @@ jobs: jupyter nbextension enable --py tensorflow_model_analysis --sys-prefix - name: Test Jupyter Notebooks - run: python test_runner.py + run: python test.py From 1f3fed5ce441c500ae9bf15000c04d97002fae2a Mon Sep 17 00:00:00 2001 From: Karan Shukla Date: Wed, 15 Apr 2020 21:49:32 -0700 Subject: [PATCH 3/9] intentional error --- .../examples/Fairness_Indicators_Example_Colab.ipynb | 1 + 1 file changed, 1 insertion(+) diff --git a/fairness_indicators/examples/Fairness_Indicators_Example_Colab.ipynb b/fairness_indicators/examples/Fairness_Indicators_Example_Colab.ipynb index 3901731e..728c3e06 100644 --- a/fairness_indicators/examples/Fairness_Indicators_Example_Colab.ipynb +++ b/fairness_indicators/examples/Fairness_Indicators_Example_Colab.ipynb @@ -83,6 +83,7 @@ "source": [ "import os\n", "import tempfile\n", + "import errormoduledoesnotexist\n", "import apache_beam as beam\n", "import numpy as np\n", "import pandas as pd\n", From ea542865a925b9354ebe4ae3164ab3993d6898dd Mon Sep 17 00:00:00 2001 From: Karan Shukla Date: Wed, 15 Apr 2020 21:50:02 -0700 Subject: [PATCH 4/9] Rename test.py to jupyter_test.py --- test.py => jupyter_test.py | 0 1 file changed, 0 insertions(+), 0 deletions(-) rename test.py => jupyter_test.py (100%) diff --git a/test.py b/jupyter_test.py similarity index 100% rename from test.py rename to jupyter_test.py From 140595a19ab178bda2785fe67fffeea38c4b64ab Mon Sep 17 00:00:00 2001 From: Karan Shukla Date: Wed, 15 Apr 2020 21:50:24 -0700 Subject: [PATCH 5/9] jupyter_test.py --- .github/workflows/jupyter.yml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.github/workflows/jupyter.yml b/.github/workflows/jupyter.yml index 301f35c3..9eb8f435 100644 --- a/.github/workflows/jupyter.yml +++ b/.github/workflows/jupyter.yml @@ -47,4 +47,4 @@ jobs: jupyter nbextension enable --py tensorflow_model_analysis --sys-prefix - name: Test Jupyter Notebooks - run: python test.py + run: python jupyter_test.py From 18387d67a704b6c1aa61a0c78281327fff26416a Mon Sep 17 00:00:00 2001 From: Karan Shukla Date: Wed, 15 Apr 2020 21:50:57 -0700 Subject: [PATCH 6/9] only do the example test --- jupyter_test.py | 58 ++++++++++++++++++++++++------------------------- 1 file changed, 29 insertions(+), 29 deletions(-) diff --git a/jupyter_test.py b/jupyter_test.py index 85265aa0..e75b16a1 100644 --- a/jupyter_test.py +++ b/jupyter_test.py @@ -42,35 +42,35 @@ def test_example(self): 'fairness_indicators/examples/Fairness_Indicators_Example_Colab.ipynb') self.assertEmpty(errors) - def test_facessd(self): - errors = notebook_runner.run_notebook( - 'fairness_indicators/examples/Facessd_Fairness_Indicators_Example_Colab.ipynb') - self.assertEmpty(errors) - - def test_lineage(self): - errors = notebook_runner.run_notebook( - 'fairness_indicators/examples/Fairness_Indicators_Lineage_Case_Study.ipynb') - self.assertEmpty(errors) - - def test_tb(self): - errors = notebook_runner.run_notebook( - 'fairness_indicators/examples/Fairness_Indicators_TensorBoard_Plugin_Example_Colab.ipynb') - self.assertEmpty(errors) - - def test_tfco_celeba(self): - errors = notebook_runner.run_notebook( - 'fairness_indicators/examples/Fairness_Indicators_TFCO_CelebA_Case_Study.ipynb') - self.assertEmpty(errors) - - def test_tfco_wiki(self): - errors = notebook_runner.run_notebook( - 'fairness_indicators/examples/Fairness_Indicators_TFCO_Wiki_Case_Study.ipynb') - self.assertEmpty(errors) - - def test_tfhub(self): - errors = notebook_runner.run_notebook( - 'fairness_indicators/examples/Fairness_Indicators_on_TF_Hub_Text_Embeddings.ipynb') - self.assertEmpty(errors) +# def test_facessd(self): +# errors = notebook_runner.run_notebook( +# 'fairness_indicators/examples/Facessd_Fairness_Indicators_Example_Colab.ipynb') +# self.assertEmpty(errors) + +# def test_lineage(self): +# errors = notebook_runner.run_notebook( +# 'fairness_indicators/examples/Fairness_Indicators_Lineage_Case_Study.ipynb') +# self.assertEmpty(errors) + +# def test_tb(self): +# errors = notebook_runner.run_notebook( +# 'fairness_indicators/examples/Fairness_Indicators_TensorBoard_Plugin_Example_Colab.ipynb') +# self.assertEmpty(errors) + +# def test_tfco_celeba(self): +# errors = notebook_runner.run_notebook( +# 'fairness_indicators/examples/Fairness_Indicators_TFCO_CelebA_Case_Study.ipynb') +# self.assertEmpty(errors) + +# def test_tfco_wiki(self): +# errors = notebook_runner.run_notebook( +# 'fairness_indicators/examples/Fairness_Indicators_TFCO_Wiki_Case_Study.ipynb') +# self.assertEmpty(errors) + +# def test_tfhub(self): +# errors = notebook_runner.run_notebook( +# 'fairness_indicators/examples/Fairness_Indicators_on_TF_Hub_Text_Embeddings.ipynb') +# self.assertEmpty(errors) if __name__ == '__main__': From 767e362985ef88448d8bbb11059de598be7f9ca7 Mon Sep 17 00:00:00 2001 From: Karan Shukla Date: Wed, 15 Apr 2020 21:56:24 -0700 Subject: [PATCH 7/9] fix _run_notebook --- jupyter_test.py | 14 +++++++------- 1 file changed, 7 insertions(+), 7 deletions(-) diff --git a/jupyter_test.py b/jupyter_test.py index e75b16a1..9b68b958 100644 --- a/jupyter_test.py +++ b/jupyter_test.py @@ -38,37 +38,37 @@ def _write_notebook(notebook): class TestNotebook(unittest.TestCase): def test_example(self): - errors = notebook_runner.run_notebook( + errors = _run_notebook( 'fairness_indicators/examples/Fairness_Indicators_Example_Colab.ipynb') self.assertEmpty(errors) # def test_facessd(self): -# errors = notebook_runner.run_notebook( +# errors = _run_notebook( # 'fairness_indicators/examples/Facessd_Fairness_Indicators_Example_Colab.ipynb') # self.assertEmpty(errors) # def test_lineage(self): -# errors = notebook_runner.run_notebook( +# errors = _run_notebook( # 'fairness_indicators/examples/Fairness_Indicators_Lineage_Case_Study.ipynb') # self.assertEmpty(errors) # def test_tb(self): -# errors = notebook_runner.run_notebook( +# errors = _run_notebook( # 'fairness_indicators/examples/Fairness_Indicators_TensorBoard_Plugin_Example_Colab.ipynb') # self.assertEmpty(errors) # def test_tfco_celeba(self): -# errors = notebook_runner.run_notebook( +# errors = _run_notebook( # 'fairness_indicators/examples/Fairness_Indicators_TFCO_CelebA_Case_Study.ipynb') # self.assertEmpty(errors) # def test_tfco_wiki(self): -# errors = notebook_runner.run_notebook( +# errors = _run_notebook( # 'fairness_indicators/examples/Fairness_Indicators_TFCO_Wiki_Case_Study.ipynb') # self.assertEmpty(errors) # def test_tfhub(self): -# errors = notebook_runner.run_notebook( +# errors = _run_notebook( # 'fairness_indicators/examples/Fairness_Indicators_on_TF_Hub_Text_Embeddings.ipynb') # self.assertEmpty(errors) From 7c37220b713d999e67269685cff05e2381b8b6ac Mon Sep 17 00:00:00 2001 From: Karan Shukla Date: Wed, 15 Apr 2020 22:03:30 -0700 Subject: [PATCH 8/9] fix assert statements --- jupyter_test.py | 15 ++++++++------- 1 file changed, 8 insertions(+), 7 deletions(-) diff --git a/jupyter_test.py b/jupyter_test.py index 9b68b958..49c331bb 100644 --- a/jupyter_test.py +++ b/jupyter_test.py @@ -40,37 +40,38 @@ class TestNotebook(unittest.TestCase): def test_example(self): errors = _run_notebook( 'fairness_indicators/examples/Fairness_Indicators_Example_Colab.ipynb') - self.assertEmpty(errors) + self.assertEqual(errors, []) + # def test_facessd(self): # errors = _run_notebook( # 'fairness_indicators/examples/Facessd_Fairness_Indicators_Example_Colab.ipynb') -# self.assertEmpty(errors) +# self.assertEqual(errors, []) # def test_lineage(self): # errors = _run_notebook( # 'fairness_indicators/examples/Fairness_Indicators_Lineage_Case_Study.ipynb') -# self.assertEmpty(errors) +# self.assertEqual(errors, []) # def test_tb(self): # errors = _run_notebook( # 'fairness_indicators/examples/Fairness_Indicators_TensorBoard_Plugin_Example_Colab.ipynb') -# self.assertEmpty(errors) +# self.assertEqual(errors, []) # def test_tfco_celeba(self): # errors = _run_notebook( # 'fairness_indicators/examples/Fairness_Indicators_TFCO_CelebA_Case_Study.ipynb') -# self.assertEmpty(errors) +# self.assertEqual(errors, []) # def test_tfco_wiki(self): # errors = _run_notebook( # 'fairness_indicators/examples/Fairness_Indicators_TFCO_Wiki_Case_Study.ipynb') -# self.assertEmpty(errors) +# self.assertEqual(errors, []) # def test_tfhub(self): # errors = _run_notebook( # 'fairness_indicators/examples/Fairness_Indicators_on_TF_Hub_Text_Embeddings.ipynb') -# self.assertEmpty(errors) +# self.assertEqual(errors, []) if __name__ == '__main__': From fc69d51560877d8b4ef2b32aa87345862008a8aa Mon Sep 17 00:00:00 2001 From: Karan Shukla Date: Wed, 15 Apr 2020 22:09:50 -0700 Subject: [PATCH 9/9] uncomment _write_notebook --- jupyter_test.py | 24 ++++++++++++------------ 1 file changed, 12 insertions(+), 12 deletions(-) diff --git a/jupyter_test.py b/jupyter_test.py index 49c331bb..94b8e5ae 100644 --- a/jupyter_test.py +++ b/jupyter_test.py @@ -8,13 +8,13 @@ def _run_notebook(path): # TODO(karanshukla): can I comment this out? - def _write_notebook(notebook): - dirname = os.path.dirname(path) - notebookname, _ = os.path.splitext(os.path.basename(path)) - output_path = os.path.join(dirname, - '{}_all_output.ipynb'.format(notebookname)) - with open(output_path, mode='wt') as f: - nbformat.write(notebook, f) +# def _write_notebook(notebook): +# dirname = os.path.dirname(path) +# notebookname, _ = os.path.splitext(os.path.basename(path)) +# output_path = os.path.join(dirname, +# '{}_all_output.ipynb'.format(notebookname)) +# with open(output_path, mode='wt') as f: +# nbformat.write(notebook, f) with open(path) as f: notebook = nbformat.read(f, as_version=4) @@ -23,7 +23,7 @@ def _write_notebook(notebook): proc.allow_errors = True proc.preprocess(notebook, {'metadata': {'path': '/'}}) - _write_notebook(notebook) +# _write_notebook(notebook) errors = [] for cell in notebook.cells: @@ -53,10 +53,10 @@ def test_example(self): # 'fairness_indicators/examples/Fairness_Indicators_Lineage_Case_Study.ipynb') # self.assertEqual(errors, []) -# def test_tb(self): -# errors = _run_notebook( -# 'fairness_indicators/examples/Fairness_Indicators_TensorBoard_Plugin_Example_Colab.ipynb') -# self.assertEqual(errors, []) + def test_tb(self): + errors = _run_notebook( + 'fairness_indicators/examples/Fairness_Indicators_TensorBoard_Plugin_Example_Colab.ipynb') + self.assertEqual(errors, []) # def test_tfco_celeba(self): # errors = _run_notebook(