Notebook Customizability¶
Customizability of notebook experiences could have accessibility implications. In this Jupyter notebook we analyze the impacts of customizability of notebooks using themes, and further analyze the various error and warning reports obtained from running accessibility scans on HTML exports of the notebooks during the execution of the pipeline.
This notebook attempts to reproduce the results in Figure 6 and Figure 7 respectively. Specifically:
- 6(b): CDF of accessibility warnings and errors based on the theme.
Required libraries:¶
collectionsmatplotlibnumpypandas
from collections import defaultdict
from helper.cdf import cdf
import matplotlib
from matplotlib.lines import Line2D
import matplotlib.pyplot as plt
import numpy
import pandas
matplotlib.rcParams.update({'font.size': 20, 'pdf.fonttype': 42, 'ps.fonttype': 42})
df = pandas.read_csv('data_out/a11y-aggregate-scan.csv')
df
Figure 6(b): CDF of Accessibility Warnings and Errors Based on Theme¶
fig_description = """Figure 6(b) indicates the CDF of the accessibility warnings and errors reported by axe and
htmlcs on various themes in our experiments. The figure contains a total of 12 colored lines, using 6 distinct
colors and two line styles. The solid lines represent the distribtion of errors per theme, and the dashed
line represents the number of warnings per theme. The solid blue and red lines indicating darcula and
light are to the extreme right side of the lines in the figure indicating the most number of accessibility
errors compared to other themes, the best performing theme is indicated by the green solid line representing
horizon theme which indicates an 85% improvement over the darcula and light themes. Other themes in our
study lie between these two lines whose medians are at 31 and 160 respectively. The x axis is represented
by a log scale indicator of the number of warnings / errors related to accessibility and the y axis indicates
the CDF ranging from 0-1.
"""
def plot_aggregate_errors_and_warnings(df):
error_results = defaultdict(list)
warning_results = defaultdict(list)
for index, row in df.iterrows():
theme = row['Theme']
error_count = row['Errors']
warning_count = row['Warnings']
error_results[theme].append(error_count)
warning_results[theme].append(warning_count)
plot_line_style = {
'warning': '--',
'error': '-',
}
theme_style = {
'darcula': '#1f77b4',
'dark': '#ff7f0e',
'horizon': '#2ca02c',
'light': '#d62728',
'material-darker': '#9467bd',
'solarized': '#8c564b',
}
fig, ax = plt.subplots(nrows=1, ncols=1, figsize=(10, 8))
for plot_type, result_dict in [('error', error_results), ('warning', warning_results)]:
for theme, arr in result_dict.items():
theme_color = theme_style[theme]
line_style = plot_line_style[plot_type]
X, Y = cdf(arr, sort_needed=True)
for p in [25, 50, 75, 90, 95, 99]:
print(f'[{plot_type}] [{theme}] [P={p}] {numpy.percentile(X, p)}')
ax.plot(X, Y, label=f'{theme}', linestyle=line_style, linewidth=2, color=theme_color)
ax.set_xlabel('Number of Warnings / Errors related to Accessibility', fontsize=18)
ax.set_ylabel('CDF of warnings/errors', fontsize=18)
ax.set_xscale('log')
line_labels = [k for k, c in theme_style.items()]
custom_lines = [Line2D([0], [0], color=c, linewidth=4) for k, c in theme_style.items()]
agg_label_lines = [Line2D([1], [1], linestyle=l, color='black', linewidth=4) for k, l in plot_line_style.items()]
agg_labels = [k for k, c in plot_line_style.items()]
l1 = ax.legend(custom_lines, line_labels, loc=2, fancybox=True, fontsize='small')
l2 = ax.legend(agg_label_lines, agg_labels, loc=4, fontsize='large')
ax.add_artist(l1)
ax.add_artist(l2)
plt.savefig('plot_out/fig-6b-warnings_errors.pdf', bbox_inches='tight')
plt.savefig('plot_out/alt-embedded-images/fig-6b-warnings_errors.png',
metadata={'alt': fig_description},
bbox_inches='tight')
plot_aggregate_errors_and_warnings(df)
Figure 6(b) indicates the CDF of the accessibility warnings and errors reported by axe and htmlcs on various themes in our experiments. The figure contains a total of 12 colored lines, using 6 distinct colors and two line styles. The solid lines represent the distribtion of errors per theme, and the dashed line represents the number of warnings per theme. The solid blue and red lines indicating darcula and light are to the extreme right side of the lines in the figure indicating the most number of accessibility errors compared to other themes, the best performing theme is indicated by the green solid line representing horizon theme which indicates an 85% improvement over the darcula and light themes. Other themes in our study lie between these two lines whose medians are at 31 and 160 respectively. The x axis is represented by a log scale indicator of the number of warnings / errors related to accessibility and the y axis indicates the CDF ranging from 0-1.