
REF1_HASH_DEFAULT = None
REF2_HASH_DEFAULT = 'upstream/master'


import os
import shutil
import tempfile
import subprocess
import numpy as np
import pandas as pd
import matplotlib.pyplot as plt
from bokeh.plotting import figure, show, ColumnDataSource
from bokeh.layouts import gridplot
from bokeh.models.tools import HoverTool
from bokeh.io import output_notebook

output_notebook()

---------------------------------------------------------------------------
ModuleNotFoundError                       Traceback (most recent call last)
<ipython-input-1-9f363699fb7c> in <module>
      6 import pandas as pd
      7 import matplotlib.pyplot as plt
----> 8 from bokeh.plotting import figure, show, ColumnDataSource
      9 from bokeh.layouts import gridplot
     10 from bokeh.models.tools import HoverTool

ModuleNotFoundError: No module named 'bokeh'


cd ..

/home/vsts/work/1/s/tardis-refdata


def highlight_missing(val):
    if val == True:
        return 'background-color: #BCF5A9'
    else:
        return 'background-color: #F5A9A9'
    
def highlight_relative_difference(val):
    ret = 'background-color: #BCF5A9'
    if val is None:
        ret = 'background-color: #BCF5A9'
    elif val > 1e-2:
        ret = 'background-color: #F2F5A9'
    elif val > 1e-1:
        ret = 'background-color: #F5D0A9'
    elif val > 1:
        ret = 'background-color: #F5A9A9'
    return ret


class ReferenceComparer(object):

    def __init__(self, ref1_hash=None, ref2_hash=None, compare_path='unit_test_data.h5'):
        assert not ((ref1_hash is None) and (ref2_hash is None)), "One hash can not be None"
        self.ref1_hash = ref1_hash
        self.ref2_hash = ref2_hash
        self.compare_path = compare_path
        self.tmp_dir = None
        self.setup()
    
    def setup(self):
        self.tmp_dir = tempfile.mkdtemp()
        print('Created temporary directory at {0}. Delete after use with .teardown'.format(self.tmp_dir))
        for ref_id, ref_hash in enumerate([self.ref1_hash, self.ref2_hash]):
            ref_id += 1
            if ref_hash is not None:
                self._copy_data_from_hash(ref_hash, 'ref{0}_'.format(ref_id))
            else:
                subprocess.Popen('cp {0} {1}'.format(self.compare_path, 
                                                     os.path.join(self.tmp_dir, 
                                                                  'ref{0}_{1}'.format(ref_id, self.compare_path))), 
                                                     shell=True)
            setattr(self, 'ref{0}_fname'.format(ref_id), 
                    os.path.join(self.tmp_dir, 'ref{0}_{1}'.format(ref_id, self.compare_path)))

    def teardown(self):
        shutil.rmtree(self.tmp_dir)

    def _copy_data_from_hash(self, ref_hash, prefix):
        git_cmd = ['git']
        git_cmd.append('--work-tree={0}'.format(self.tmp_dir))
        git_cmd += ['checkout', ref_hash, self.compare_path]
        p = subprocess.Popen(git_cmd)
        p.wait()
        shutil.move(os.path.join(self.tmp_dir, self.compare_path), 
                    os.path.join(self.tmp_dir, prefix + self.compare_path))

    def generate_test_table(self):
        rd1_hdfs = pd.HDFStore(self.ref1_fname, mode='r')
        rd2_hdfs = pd.HDFStore(self.ref2_fname, mode='r')
        rd1_keys = rd1_hdfs.keys()
        rd2_keys = rd2_hdfs.keys()
        rd1_hdfs.close()
        rd2_hdfs.close()
        rd1_df = pd.DataFrame(index=rd1_keys, columns=['exists'])
        rd2_df = pd.DataFrame(index=rd2_keys, columns=['exists'])
        rd1_df['exists'] = True
        rd2_df['exists'] = True
        joined_df = rd1_df.join(rd2_df, how='outer', lsuffix='_1', rsuffix='_2')
        joined_df = joined_df.fillna(False)
        return joined_df
    
    def compare_refdata(self, test_table):
        test_table['match'] = None
        test_table['abs_diff_mean'] = None
        test_table['abs_diff_max'] = None
        test_table['rel_diff_mean'] = None
        test_table['rel_diff_max'] = None
        for row_id, row in test_table.iterrows():
            
            if row[['exists_1', 'exists_2']].all():
                ref1_df = pd.read_hdf(self.ref1_fname, row_id)
                ref2_df = pd.read_hdf(self.ref2_fname, row_id)
                
                if isinstance(ref1_df, pd.Series):
                    try:
                        pd.testing.assert_series_equal(ref1_df, ref2_df)
                    except AssertionError:
                        test_table.loc[row_id, 'match'] = False
                        abs_diff = np.fabs(ref1_df - ref2_df)
                        rel_diff = (abs_diff / np.fabs(ref1_df))[ref1_df != 0]
                        test_table.loc[row_id, 'abs_diff_mean'] = abs_diff.mean()
                        test_table.loc[row_id, 'abs_diff_max'] = abs_diff.max()
                        test_table.loc[row_id, 'rel_diff_mean'] = rel_diff.mean()
                        test_table.loc[row_id, 'rel_diff_max'] = rel_diff.max()
                    else:
                        test_table.loc[row_id, 'match'] = True

                elif isinstance(ref1_df, pd.DataFrame):
                    try:
                        pd.testing.assert_frame_equal(ref1_df, ref2_df)
                    except AssertionError:
                        test_table.loc[row_id, 'match'] = False
                        abs_diff = np.fabs(ref1_df - ref2_df)
                        rel_diff = (abs_diff / np.fabs(ref1_df))[ref1_df != 0]
                        test_table.loc[row_id, 'abs_diff_mean'] = abs_diff.mean(skipna=True).mean()
                        test_table.loc[row_id, 'abs_diff_max'] = abs_diff.max(skipna=True).max()
                        test_table.loc[row_id, 'rel_diff_mean'] = rel_diff.mean(skipna=True).mean()
                        test_table.loc[row_id, 'rel_diff_max'] = rel_diff.max(skipna=True).max()
                    else:
                        test_table.loc[row_id, 'match'] = True

                else:
                    raise ValueError('Needs to be a Series or DataFrame but is' + str(type(ref1_df)))
        return test_table


try:
    REF1_HASH = os.environ['REF1_HASH']

    if not REF1_HASH:
        raise ValueError

except (KeyError, ValueError):
    REF1_HASH = REF1_HASH_DEFAULT

try:
    REF2_HASH = os.environ['REF2_HASH']

    if not REF2_HASH:
        raise ValueError

except (KeyError, ValueError):
    REF2_HASH = REF2_HASH_DEFAULT


REF1_HASH, REF2_HASH

(None, 'upstream/master')


comparer = ReferenceComparer(ref1_hash=REF1_HASH, ref2_hash=REF2_HASH)

Created temporary directory at /tmp/tmpw2trg9l4. Delete after use with .teardown


tt = comparer.generate_test_table()


tt = comparer.compare_refdata(tt)

---------------------------------------------------------------------------
ValueError                                Traceback (most recent call last)
<ipython-input-1-197587a6f814> in <module>
----> 1 tt = comparer.compare_refdata(tt)

<ipython-input-1-aa05e4877f43> in compare_refdata(self, test_table)
     60 
     61             if row[['exists_1', 'exists_2']].all():
---> 62                 ref1_df = pd.read_hdf(self.ref1_fname, row_id)
     63                 ref2_df = pd.read_hdf(self.ref2_fname, row_id)
     64 

/usr/share/miniconda/envs/tardis/lib/python3.7/site-packages/pandas/io/pytables.py in read_hdf(path_or_buf, key, mode, errors, where, start, stop, columns, iterator, chunksize, **kwargs)
    426             iterator=iterator,
    427             chunksize=chunksize,
--> 428             auto_close=auto_close,
    429         )
    430     except (ValueError, TypeError, KeyError):

/usr/share/miniconda/envs/tardis/lib/python3.7/site-packages/pandas/io/pytables.py in select(self, key, where, start, stop, columns, iterator, chunksize, auto_close)
    812         )
    813 
--> 814         return it.get_result()
    815 
    816     def select_as_coordinates(

/usr/share/miniconda/envs/tardis/lib/python3.7/site-packages/pandas/io/pytables.py in get_result(self, coordinates)
   1827 
   1828         # directly return the result
-> 1829         results = self.func(self.start, self.stop, where)
   1830         self.close()
   1831         return results

/usr/share/miniconda/envs/tardis/lib/python3.7/site-packages/pandas/io/pytables.py in func(_start, _stop, _where)
    796         # function to call on iteration
    797         def func(_start, _stop, _where):
--> 798             return s.read(start=_start, stop=_stop, where=_where, columns=columns)
    799 
    800         # create the iterator

/usr/share/miniconda/envs/tardis/lib/python3.7/site-packages/pandas/io/pytables.py in read(self, where, columns, start, stop)
   3068 
   3069             blk_items = self.read_index(f"block{i}_items")
-> 3070             values = self.read_array(f"block{i}_values", start=_start, stop=_stop)
   3071 
   3072             columns = items[items.get_indexer(blk_items)]

/usr/share/miniconda/envs/tardis/lib/python3.7/site-packages/pandas/io/pytables.py in read_array(self, key, start, stop)
   2723 
   2724         if isinstance(node, tables.VLArray):
-> 2725             ret = node[0][start:stop]
   2726         else:
   2727             dtype = getattr(attrs, "value_type", None)

/usr/share/miniconda/envs/tardis/lib/python3.7/site-packages/tables/vlarray.py in __getitem__(self, key)
    675                 key += self.nrows
    676             (start, stop, step) = self._process_range(key, key + 1, 1)
--> 677             return self.read(start, stop, step)[0]
    678         elif isinstance(key, slice):
    679             start, stop, step = self._process_range(

/usr/share/miniconda/envs/tardis/lib/python3.7/site-packages/tables/vlarray.py in read(self, start, stop, step)
    819         atom = self.atom
    820         if not hasattr(atom, 'size'):  # it is a pseudo-atom
--> 821             outlistarr = [atom.fromarray(arr) for arr in listarr]
    822         else:
    823             # Convert the list to the right flavor

/usr/share/miniconda/envs/tardis/lib/python3.7/site-packages/tables/vlarray.py in <listcomp>(.0)
    819         atom = self.atom
    820         if not hasattr(atom, 'size'):  # it is a pseudo-atom
--> 821             outlistarr = [atom.fromarray(arr) for arr in listarr]
    822         else:
    823             # Convert the list to the right flavor

/usr/share/miniconda/envs/tardis/lib/python3.7/site-packages/tables/atom.py in fromarray(self, array)
   1222         if array.size == 0:
   1223             return None
-> 1224         return pickle.loads(array.tostring())

ValueError: unsupported pickle protocol: 5


tt[["exists_1", "exists_2", 'rel_diff_mean', 'rel_diff_max', 'match']].style.applymap(
    highlight_missing, subset=['exists_1', 'exists_2', 'match']).applymap(
    highlight_relative_difference, subset=['rel_diff_mean', 'rel_diff_max'])


def compare_output_nu(df1, df2, mpl_backend=False):
    nu_min = np.min([df1.min(), df2.min()])
    nu_max = np.max([df1.max(), df2.max()])
    
    if mpl_backend:
        plt.figure(figsize=(14, 6))
        plt.subplot(121)
        plt.plot(df1, df2, ',')
        plt.xlabel("output_nu, ref 1")
        plt.ylabel("output_nu, ref 2")
        plt.subplot(122)
        plt.hist(df1, bins=np.linspace(nu_min, nu_max, 100), histtype="step", label="ref 1")
        plt.hist(df2, bins=np.linspace(nu_min, nu_max, 100), histtype="step", label="ref 2")
        plt.xlabel("output_nu")
        plt.legend(frameon=False)
        
        return

    
    TOOLTIPS = [("(x,y)", "(@x, @y)")]
    hover = HoverTool(tooltips=TOOLTIPS)
    
    p = figure()
    output_nu = ColumnDataSource(pd.DataFrame.from_records({'x': df1.values, 
                                                            'y': df2.values}))
    p.circle('x', 'y', size=1, source=output_nu)
    p.xaxis.axis_label = "output_nu, ref 1"
    p.yaxis.axis_label = "output_nu, ref 2"
    p.xaxis.formatter.precision = 1
    p.yaxis.formatter.precision = 1
    p.add_tools(hover)

    # Step lines are hacky way to make histograms with Bokeh
    arr_hist1, edges1 = np.histogram(df1.values, 
                                     bins = 100, 
                                     range = [nu_min, nu_max])
    arr_hist2, edges2 = np.histogram(df1.values, 
                                     bins = 100, 
                                     range = [nu_min, nu_max])
    
    hist1 = ColumnDataSource(pd.DataFrame.from_records({'x': np.linspace(nu_min, nu_max, 100),
                                                        'y': arr_hist1}))
    hist2 = ColumnDataSource(pd.DataFrame.from_records({'x': np.linspace(nu_min, nu_max, 100),
                                                        'y': arr_hist2}))
    q = figure()
    q.step('x', 'y', source=hist1, legend_label='ref 1')
    q.step('x', 'y', source=hist2, legend_label='ref 2', color='#ff7f0e')
    q.xaxis.axis_label = "output_nu"
    q.xaxis.formatter.precision = 1
    q.legend.click_policy="hide"
    
    # Currently HoverTool does not work for step line glyph. See: https://github.com/bokeh/bokeh/issues/7419
    q.add_tools(hover)
    
    plot = gridplot([p, q], ncols=2, plot_width=420, plot_height=360)
    
    show(plot)


def compare_spectrum(ref1_nu, ref1_L, ref2_nu, ref2_L, mpl_backend=False):
    
    if mpl_backend:
        plt.figure(figsize=(14, 6))
        plt.subplot(121)
        plt.plot(ref1_nu, ref1_L, label="ref 1")
        plt.plot(ref2_nu, ref2_L, label="ref 2")
        plt.xlabel("nu")
        plt.ylabel("L")
        plt.legend(frameon=False)
        plt.subplot(122)
        plt.plot(ref1_nu, ref1_L / ref2_L)
        plt.xlabel("nu")
        plt.ylabel("L ref 1 / L ref 2")
        
        return
    
    
    TOOLTIPS = [("(x,y)", "(@x, @y)")]
    hover = HoverTool(tooltips=TOOLTIPS)
    
    p = figure()
    spectrum1 = ColumnDataSource(pd.DataFrame.from_records({'x': ref1_nu.values, 
                                                            'y': ref1_L}))
    spectrum2 = ColumnDataSource(pd.DataFrame.from_records({'x': ref2_nu.values, 
                                                            'y': ref2_L}))
    p.line('x', 'y', source=spectrum1, legend_label='ref 1')
    p.line('x', 'y', source=spectrum2, legend_label='ref 2', color='#ff7f0e')
    p.xaxis.axis_label = "L"
    p.yaxis.axis_label = "nu"
    p.xaxis.formatter.precision = 1
    p.yaxis.formatter.precision = 1
    p.legend.click_policy="hide"
    p.add_tools(hover)
    
    q = figure()
    lum_ratio = ColumnDataSource(pd.DataFrame.from_records({'x': ref1_nu.values, 
                                                            'y': ref1_L.values/ref2_L.values}))
    q.circle('x', 'y', size=1, source=lum_ratio)
    q.xaxis.axis_label = "nu"
    q.yaxis.axis_label = "L ref 1 / L ref 2"
    q.xaxis.formatter.precision = 1
    q.yaxis.formatter.precision = 1
    q.add_tools(hover)
    
    
    plot = gridplot([p, q], ncols=2, plot_width=420, plot_height=360)
    
    show(plot)


tmp1 = pd.HDFStore(comparer.ref1_fname, "r")
tmp2 = pd.HDFStore(comparer.ref2_fname, "r")

diff_entries = tt.loc[(tt["match"] == False) & (tt["exists_1"] == True) & (tt["exists_2"] == True)].index


compare_output_nu(tmp1['/test_simulation/output_nu'], tmp2['/test_simulation/output_nu'])

---------------------------------------------------------------------------
NameError                                 Traceback (most recent call last)
<ipython-input-1-54d5fb0ce665> in <module>
----> 1 compare_output_nu(tmp1['/test_simulation/output_nu'], tmp2['/test_simulation/output_nu'])

<ipython-input-1-448903146483> in compare_output_nu(df1, df2, mpl_backend)
     19 
     20     TOOLTIPS = [("(x,y)", "(@x, @y)")]
---> 21     hover = HoverTool(tooltips=TOOLTIPS)
     22 
     23     p = figure()

NameError: name 'HoverTool' is not defined


compare_spectrum(tmp1['/test_runner_simple/spectrum/_frequency'][:-1], 
                 tmp1['/test_runner_simple/spectrum/luminosity'],
                 tmp2['/test_runner_simple/spectrum/_frequency'][:-1], 
                 tmp2['/test_runner_simple/spectrum/luminosity'])

---------------------------------------------------------------------------
NameError                                 Traceback (most recent call last)
<ipython-input-1-0257c581a562> in <module>
      2                  tmp1['/test_runner_simple/spectrum/luminosity'],
      3                  tmp2['/test_runner_simple/spectrum/_frequency'][:-1],
----> 4                  tmp2['/test_runner_simple/spectrum/luminosity'])

<ipython-input-1-95c65f3b7892> in compare_spectrum(ref1_nu, ref1_L, ref2_nu, ref2_L, mpl_backend)
     18 
     19     TOOLTIPS = [("(x,y)", "(@x, @y)")]
---> 20     hover = HoverTool(tooltips=TOOLTIPS)
     21 
     22     p = figure()

NameError: name 'HoverTool' is not defined


comparer.teardown()

Compare reference data¶

Installation¶

Usage¶

Define classes and functions¶

Load data¶

Test table¶

Detailed inspection of the reference data¶

Results¶