Browse Source

Syntax highlighting via pygments

tags/v5.5.4
Johannes Köster 2 months ago
parent
commit
aaa2138a37
4 changed files with 62 additions and 29 deletions
  1. 1
    1
      setup.py
  2. 30
    7
      snakemake/report/__init__.py
  3. 2
    1
      snakemake/report/report.html
  4. 29
    20
      snakemake/script.py

+ 1
- 1
setup.py View File

@@ -49,7 +49,7 @@ setup(
install_requires=['wrapt', 'requests', 'ratelimiter', 'pyyaml',
'configargparse', 'appdirs', 'datrie', 'jsonschema',
'docutils', 'gitpython'],
extras_require={"reports": ['jinja2', 'networkx']},
extras_require={"reports": ['jinja2', 'networkx', 'pygments']},
classifiers=
["Development Status :: 5 - Production/Stable", "Environment :: Console",
"Intended Audience :: Science/Research",

+ 30
- 7
snakemake/report/__init__.py View File

@@ -184,7 +184,6 @@ class Category:


class RuleRecord:
code = dict()

def __init__(self, job, job_rec):
import yaml
@@ -201,14 +200,30 @@ class RuleRecord:
self.id = uuid.uuid4()

def code(self):
try:
from pygments.lexers import get_lexer_by_name
from pygments.formatters import HtmlFormatter
from pygments import highlight
except ImportError:
raise WorkflowError("Python package pygments must be installed to create reports.")
source, language = None, None
if self._rule.shellcmd is not None:
return self._rule.shellcmd
source = self._rule.shellcmd
language = "bash"
elif self._rule.script is not None:
logger.info("Loading script code for rule {}".format(self.name))
return script.get_source(self._rule.script, job.rule.basedir)
elif self._rule.wrapper:
_, source, language = script.get_source(self._rule.script, self._rule.basedir)
source = source.decode()
elif self._rule.wrapper is not None:
logger.info("Loading wrapper code for rule {}".format(self.name))
return script.get_source(wrapper.get_script(self._rule.wrapper, prefix=self._rule.workflow.wrapper_prefix), self._rule.basedir)
_, source, language = script.get_source(wrapper.get_script(self._rule.wrapper, prefix=self._rule.workflow.wrapper_prefix))
source = source.decode()

try:
lexer = get_lexer_by_name(language)
return highlight(source, lexer, HtmlFormatter(linenos=True, cssclass="source", wrapcode=True))
except pygments.utils.ClassNotFound:
return "<pre><code>source</code></pre>"

def add(self, job_rec):
self.n_jobs += 1
@@ -222,6 +237,7 @@ class RuleRecord:

class JobRecord:
def __init__(self):
self.job = None
self.rule = None
self.starttime = sys.maxsize
self.endtime = 0
@@ -433,6 +449,7 @@ def auto_report(dag, path):
rule = meta["rule"]
rec = records[(job_hash, rule)]
rec.rule = rule
rec.job = job
rec.starttime = min(rec.starttime, meta["starttime"])
rec.endtime = max(rec.endtime, meta["endtime"])
rec.conda_env_file = None
@@ -460,7 +477,7 @@ def auto_report(dag, path):
# prepare per-rule information
rules = defaultdict(list)
for rec in records.values():
rule = RuleRecord(job, rec)
rule = RuleRecord(rec.job, rec)
if rec.rule not in rules:
rules[rec.rule].append(rule)
else:
@@ -513,6 +530,11 @@ def auto_report(dag, path):
now = "{} {}".format(datetime.datetime.now().ctime(), time.tzname[0])
results_size = sum(res.size for cat in results.values() for res in cat)

try:
from pygments.formatters import HtmlFormatter
except ImportError:
raise WorkflowError("Python package pygments must be installed to create reports.")

# render HTML
template = env.get_template("report.html")
with open(path, "w", encoding="utf-8") as out:
@@ -527,5 +549,6 @@ def auto_report(dag, path):
timeline=timeline,
rules=[rec for recs in rules.values() for rec in recs],
version=__version__,
now=now))
now=now,
pygments_css=HtmlFormatter(style="trac").get_style_defs('.source')))
logger.info("Report created.")

+ 2
- 1
snakemake/report/report.html View File

@@ -12,6 +12,7 @@
<style>{{ "https://stackpath.bootstrapcdn.com/bootstrap/4.1.1/css/bootstrap.min.css"|get_resource_as_string }}</style>
<style>{{ "https://cdnjs.cloudflare.com/ajax/libs/ekko-lightbox/5.3.0/ekko-lightbox.css"|get_resource_as_string }}</style>
<style>{{ "https://cdn.datatables.net/v/bs4/dt-1.10.18/r-2.2.2/datatables.min.css"|get_resource_as_string }}</style>
<style>{{ pygments_css|safe }}</style>

<!-- Custom styles for this template -->
<style>
@@ -373,7 +374,7 @@
{% if rule.code is not none %}
<a data-toggle="collapse" role="button" href="#code-{{ rule.id }}" aria-expanded="false" aria-controls="collapse-env"><span data-feather="plus-circle"></span></a>
<div class="collapse" id="code-{{ rule.id }}">
<pre class="pre-scrollable">{{ rule.code() }}</pre>
{{ rule.code()|safe }}
</div>
{% endif %}
</td>

+ 29
- 20
snakemake/script.py View File

@@ -227,7 +227,8 @@ class Snakemake:
return lookup[(stdout, stderr, append)].format(self.log)


def get_source(path, basedir):
def get_source(path, basedir=None):
source = None
if not path.startswith("http") and not path.startswith("git+file"):
if path.startswith("file://"):
path = path[7:]
@@ -238,21 +239,29 @@ def get_source(path, basedir):
path = "file://" + path
path = format(path, stepout=1)
if path.startswith("file://"):
sourceurl = "file:"+pathname2url(path[7:])
sourceurl = "file:" + pathname2url(path[7:])
elif path.startswith("git+file"):
source = git_content(path)
(root_path, file_path, version) = split_git_path(path)
dir = ".snakemake/wrappers"
os.makedirs(dir, exist_ok=True)
new_path = os.path.join(dir, version + "-"+ "-".join(file_path.split("/")))
with open(new_path,'w') as wrapper:
wrapper.write(git_content(path))
sourceurl = "file:" + new_path
path = path.rstrip("@" + version)
path = path.rstrip("@" + version)
else:
sourceurl = path
with urlopen(sourceurl) as source:
return path, source.read()
language = None
if path.endswith(".py"):
language = "python"
elif path.endswith(".R"):
language = "r"
elif path.endswith(".Rmd"):
language = "rmarkdown"
elif path.endswith(".jl"):
language = "julia"
if source is None:
with urlopen(sourceurl) as source:
return path, source.read(), language
else:
return path, source, language


def script(path, basedir, input, output, params, wildcards, threads, resources,
@@ -265,8 +274,8 @@ def script(path, basedir, input, output, params, wildcards, threads, resources,

f = None
try:
path, source = get_source(path, basedir)
if path.endswith(".py"):
path, source, language = get_source(path, basedir)
if language == "python":
wrapper_path = path[7:] if path.startswith("file://") else path
snakemake = Snakemake(input, output, params, wildcards,
threads, resources, log, config, rulename,
@@ -292,7 +301,7 @@ def script(path, basedir, input, output, params, wildcards, threads, resources,
snakemake=snakemake,
printshellcmds=logger.printshellcmds,
file_override=repr(os.path.realpath(wrapper_path)))
elif path.endswith(".R") or path.endswith(".Rmd"):
elif language == "r" or language == "rmarkdown":
preamble = textwrap.dedent("""
######## Snakemake header ########
library(methods)
@@ -347,7 +356,7 @@ def script(path, basedir, input, output, params, wildcards, threads, resources,
}), REncoder.encode_dict(config), REncoder.encode_value(rulename),
REncoder.encode_numeric(bench_iteration),
REncoder.encode_value(os.path.dirname(path[7:]) if path.startswith("file://") else os.path.dirname(path)))
elif path.endswith(".jl"):
elif language == "julia":
preamble = textwrap.dedent("""
######## Snakemake header ########
struct Snakemake
@@ -408,7 +417,7 @@ def script(path, basedir, input, output, params, wildcards, threads, resources,
suffix="." + os.path.basename(path),
dir=dir,
delete=False) as f:
if not path.endswith(".Rmd"):
if not language == "rmarkdown":
f.write(preamble.encode())
f.write(source)
else:
@@ -424,7 +433,7 @@ def script(path, basedir, input, output, params, wildcards, threads, resources,
f.write(preamble.encode())
f.write(str.encode(code[pos:]))

if path.endswith(".py"):
if language == "python":
py_exec = sys.executable
if conda_env is not None:
py = os.path.join(conda_env, "bin", "python")
@@ -450,7 +459,7 @@ def script(path, basedir, input, output, params, wildcards, threads, resources,
py_exec = "python"
# use the same Python as the running process or the one from the environment
shell("{py_exec} {f.name:q}", bench_record=bench_record)
elif path.endswith(".R"):
elif language == "r":
if conda_env is not None and "R_LIBS" in os.environ:
logger.warning("R script job uses conda environment but "
"R_LIBS environment variable is set. This "
@@ -460,14 +469,14 @@ def script(path, basedir, input, output, params, wildcards, threads, resources,
"remove it entirely before executing "
"Snakemake.")
shell("Rscript --vanilla {f.name:q}", bench_record=bench_record)
elif path.endswith(".Rmd"):
elif language == "rmakrdown":
if len(output) != 1:
raise WorkflowError("RMarkdown scripts (.Rmd) may only have a single output file.")
out = os.path.abspath(output[0])
shell("Rscript --vanilla -e 'rmarkdown::render(\"{f.name}\", output_file=\"{out}\", quiet=TRUE, knit_root_dir = \"{workdir}\", params = list(rmd=\"{f.name}\"))'",
bench_record=bench_record,
workdir=os.getcwd())
elif path.endswith(".jl"):
elif language == "julia":
shell("julia {f.name:q}", bench_record=bench_record)

except URLError as e:

Loading…
Cancel
Save