Newer
Older
gradle-lectures / buildSrc / src / main / java / nz / stanger / lecture / Lecture.java
package nz.stanger.lecture;

import java.io.File;
import java.util.ArrayList;
import java.util.Arrays;
import java.util.Collection;
import java.util.List;
import java.util.HashMap;
import java.util.HashSet;
import java.util.Map;
import java.util.Set;
import java.util.stream.Collectors;

import org.danilopianini.gradle.latex.LatexExtension;

import org.gradle.api.Plugin;
import org.gradle.api.Project;
import org.gradle.api.file.ConfigurableFileTree;
import org.gradle.api.file.Directory;
import org.gradle.api.provider.Provider;
import org.gradle.api.provider.SetProperty;
import org.gradle.api.tasks.Copy;
import org.gradle.api.tasks.Delete;
import org.gradle.api.tasks.Exec;
import org.gradle.api.tasks.TaskContainer;

public class Lecture implements Plugin<Project> {

    private static final String CONTENT = "content";
    private static final String SLIDES = "slides";
    private static final String HANDOUT = "handout";
    private static final String NOTES = "notes";
    private static final String EXAMPLES = "examples";

    private static final String LECTURE_PREFIX = "lecture";

    private static final String TASK_GROUP = "Lecture";

    private static final Set<String> VALID_TARGET_NAMES = Set.of(
            SLIDES,
            HANDOUT,
            NOTES,
            EXAMPLES
    );

    // Some handy references that are used a lot.
    private TaskContainer projectTasks;
    private Directory projectDirectory;
    private LectureExtension lecture;
    private LatexExtension latex;

    /**
     * Find directories whose names match a regular expression.
     *
     * Adapted from https://gist.github.com/Ethsaam/2b8dd748fcc25e6973bb.
     *
     * @param dir the directory to start the search in
     * @param pattern regular expression matching directory names
     * @return the collection of matching directories
     */
    private Collection<File> getListOfDirectories(Directory dir, String pattern) {
        return new ArrayList<>(Arrays.asList(dir.getAsFile().listFiles(file -> file.isDirectory() && file.getName().matches(pattern))));
    }

    /**
     * Register a task to generate an image from some other format.
     *
     * @param srcRegex regular expression matching source files (inputs)
     * @param dstRegex regular expression matching destination files (outputs)
     * @param cmdTemplate list of command line components (strings)
     */
    private void registerGeneratedImageTask(String srcRegex, String dstRegex, List<String> cmdTemplate) {
        lecture.getGeneratedImages().get().entrySet().stream()
                .filter(entry -> entry.getValue().matches(srcRegex))
                .filter(entry -> entry.getKey().matches(dstRegex))
                .forEach(entry -> {
                    File dst = lecture.getImageDir().file(entry.getKey()).get().getAsFile();
                    File src = lecture.getImageDir().file(entry.getValue()).get().getAsFile();
                    projectTasks.register("images." + entry.getKey(), Exec.class, task -> {
                        task.setDescription("Generate " + entry.getKey() + " from " + entry.getValue() + ".");
                        task.setGroup(TASK_GROUP);
                        File cropped = lecture.getImageDir().file(entry.getKey().replace(".pdf", "-crop.pdf")).get().getAsFile();

                        // set working directory to imageDir so that relative
                        // references work, e.g., PlantUML includes
                        task.setWorkingDir(lecture.getImageDir().get());
                        task.getInputs().files(src);
                        task.getOutputs().files(dst);

                        List<String> command = cmdTemplate.stream().map(
                                part -> part.replaceAll("<<src>>", src.getName())
                                        .replaceAll("<<dst>>", dst.getName())
                                        .replaceAll("<<cropped>>", cropped.getName())
                        ).collect(Collectors.toList());
                        task.commandLine(command);
                    });
                    // Every PDF generation will have a different hash, so only run the
                    // task if the input is newer than the output.
                    projectTasks.getByPath("images." + entry.getKey()).onlyIf(t -> {
                        return src.lastModified() > dst.lastModified();
                    });
                });
    }

    @Override
    public void apply(Project project) {
        projectTasks = project.getTasks();
        projectDirectory = project.getLayout().getProjectDirectory();

        project.getPluginManager().apply("org.danilopianini.gradle-latex");

        // Plugin extensions.
        lecture = project.getExtensions().create("lecture", LectureExtension.class);
        latex = project.getExtensions().getByType(LatexExtension.class);

        // Configure LaTeX extension.
        latex.getPdfLatexCommand().set("xelatex");

        /**
         * Set plugin defaults.
         */
        // Supported document targets.
        lecture.getDocTargets().convention(VALID_TARGET_NAMES);

        // Validate document target configuration. Must be afterEvaluate() to
        // ensure everything has been finalised.
        project.afterEvaluate(p -> {
            lecture.getDocTargets().get().stream().forEach(target -> {
                if (!VALID_TARGET_NAMES.contains(target)) {
                    throw new RuntimeException("unsupported target name " + target);
                }
            });
        });

        // Supported targets (numbered documents). Automatically derived from
        // LectureExtension.docTargets.
        Provider<Set<String>> numberedDocTargets = lecture.getDocTargets().map(
                docTargets -> docTargets.stream().map(target -> target + ".numbered").collect(Collectors.toSet())
        );

        // Standard lecture prefix string.
        lecture.getDocPrefix().convention(LECTURE_PREFIX);

        lecture.getLectureNum().convention(project.getName().substring(0, 2));

        // Standard document names (unnumbered). Automatically derived from
        // LectureExtension.docTargets, plus "content".
        Provider<Map<String, String>> docNames = lecture.getDocTargets().map(docTargets -> {
            Map<String, String> names = new HashMap<>();
            docTargets.forEach(target -> names.put(target, lecture.getDocPrefix().get() + "_" + target));
            names.put(CONTENT, lecture.getDocPrefix().get() + "_" + CONTENT);
            return names;
        });

        // Standard document names (numbered). Automatically derived from
        // LectureExtension.docTargets, plus "content".
        Provider<Map<String, String>> numberedDocNames = lecture.getDocTargets().map(docTargets -> {
            Map<String, String> names = new HashMap<>();
            docTargets.forEach(target -> names.put(target, lecture.getDocPrefix().get() + "_" + lecture.getLectureNum().get() + "_" + target));
            names.put(CONTENT, lecture.getDocPrefix().get() + "_" + lecture.getLectureNum().get() + "_" + CONTENT);
            return names;
        });

        lecture.getImageDir().convention(projectDirectory.dir("images"));

        lecture.getPdfDir().convention(projectDirectory.dir("pdfs"));

        // slidesImages defaults to empty
        lecture.getSlidesFiles().convention(
                lecture.getDocPrefix().map(
                        (String prefix) -> Set.of(
                                prefix + "_" + CONTENT + ".tex",
                                "paper_init.tex",
                                "lecturedates.tex",
                                "doc_init.tex"
                        )
                )
        );

        // examplesImages defaults to empty
        lecture.getExamplesFiles().convention(
                lecture.getDocPrefix().map((String prefix) -> {
                    if (lecture.getDocTargets().get().contains(EXAMPLES)) {
                        return Set.of(
                                prefix + "_" + EXAMPLES + ".tex",
                                "paper_init.tex",
                                "lecturedates.tex",
                                "doc_init.tex"
                        );
                    } else {
                        return Set.of();
                    }
                })
        );

        // Document dependencies. Automatically derived from slidesImages,
        // slidesFiles, examplesImages, and examplesFiles. Images are mapped to
        // imageDir. The slides, handout, and notes will always share the same
        // dependencies. Only the examples will be different.
        Provider<Set<String>> slideDependencies = lecture.getSlidesImages().map(slideImages -> {
            Set<String> slideDeps = new HashSet<>();
            slideDeps.addAll(slideImages.stream().map(image -> lecture.getImageDir().get() + "/" + image).collect(Collectors.toSet()));
            slideDeps.addAll(lecture.getSlidesFiles().get());
            return slideDeps;
        });

        Provider<Map<String, List<String>>> docDependencies = lecture.getDocTargets().map(docTargets -> {
            Map<String, List<String>> docDeps = new HashMap<>();
            docTargets.forEach(target -> {
                if (target.equals("examples")) {
                    List<String> deps = lecture.getExamplesFiles().get().stream().collect(Collectors.toList());
                    deps.addAll(lecture.getExamplesImages().get().stream()
                            .map(image -> lecture.getImageDir().get() + "/" + image)
                            .collect(Collectors.toList())
                    );
                    docDeps.put(target, deps);
                } else {
                    docDeps.put(target, slideDependencies.get().stream().collect(Collectors.toList()));
                }
            });
            return docDeps;
        });

        // generatedImages defaults to empty
        lecture.getCleanFiles().convention(Set.of(
                "*.aux",
                "*.dvi",
                "*.fdb_latexmk",
                "*.fls",
                "*.head",
                "*.listing",
                "*.log",
                "*.nav",
                "*.out",
                "*.pyg",
                "*.snm",
                "*.synctex.gz",
                "*.tmp",
                "*.toc",
                "*.vrb",
                "*.xdv"
        ));
        // These are regular expressions, NOT glob patterns.
        lecture.getCleanDirs().convention(Set.of(
                "_minted.*"
        ));

        lecture.getLatexFlags().convention(Set.of(
                "-shell-escape",
                "-synctex=1",
                "-interaction=nonstopmode",
                "-halt-on-error",
                "-file-line-error"
        ));

        /**
         * Tasks.
         *
         * It doesn't always seem to be strictly necessary, but it seems safer
         * in general to assume that all tasks need to be afterEvaluate(). It's
         * certainly necessary for generated tasks that derive from
         * configuration variables.
         */
        project.afterEvaluate(p -> {

            project.setDefaultTasks(Arrays.asList("docs", "numbered"));

            // All unnumbered documents.
            projectTasks.register("docs", task -> {
                task.setDescription("Build all documents.");
                task.setGroup(TASK_GROUP);
                task.dependsOn(lecture.getDocTargets());
            });

            // Generate tasks for each of the unnumbered document targets.
            // All generated tasks that derive from configuration variables must be
            // afterEvaluate() to ensure everything has been finalised.
            lecture.getDocTargets().get().forEach(target -> {
                projectTasks.register(target, task -> {
                    task.setDescription("Build " + target + " document.");
                    task.setGroup(TASK_GROUP);
                    task.dependsOn("pdfLatex." + docNames.get().get(target));
                });
            });

            // All numbered documents.
            projectTasks.register("numbered", task -> {
                task.setDescription("Create numbered versions of all documents.");
                task.setGroup(TASK_GROUP);
                task.dependsOn(numberedDocTargets);
            });

            // Generate tasks for each of the numbered document targets.
            numberedDocTargets.get().forEach(target -> {
                projectTasks.register(target, Copy.class, task -> {
                    String baseTarget = target.replace(".numbered", "");
                    task.setDescription("Build numbered version of " + baseTarget + " document.");
                    task.setGroup(TASK_GROUP);

                    task.from(project.getTasksByName("pdfLatex." + docNames.get().get(baseTarget), true));
                    task.into(lecture.getPdfDir());

                    // The LaTeX tasks include the .aux file in their outputs.
                    // Exclude or they will also be copied into the PDFs directory.
                    task.exclude("*.aux");
                    task.rename(src -> src.replace("_", "_" + lecture.getLectureNum().get() + "_"));
                });
            });

            // Generate LaTeX build tasks for each of the targets. This uses the
            // string invocation method supplied by the LaTeX plugin.
            lecture.getDocTargets().get().forEach(target -> {
                latex.invoke(docNames.get().get(target), (dsl) -> {
                    dsl.setExtraArguments(lecture.getLatexFlags().get());
                    dsl.setWatching(docDependencies.get().get(target));
                    return null; // workaround for Kotlin Unit (void) return
                });
            });

            // Add image task dependencies to the generated LaTeX tasks.
            // No need to check whether there are any generated images, Gradle does
            // the right thing if the list is empty. This also makes *all* LaTeX
            // documents dependent on *all* generated images, even when a document
            // doesn't actually depend on a specific image. This would be too much
            // of a hassle to fix given the relatively small benefit.
            projectTasks.matching(task -> task.getName().startsWith("pdfLatex")).forEach(pdfLatexTask -> {
                pdfLatexTask.dependsOn(projectTasks.matching(t -> t.getName().startsWith("images.")));
            });

            // All generated images.
            projectTasks.register("images", task -> {
                task.setDescription("Generate images.");
                task.setGroup(TASK_GROUP);
                task.setDependsOn(projectTasks.matching(t -> t.getName().startsWith("images.")));
            });

            // Remove files specified in cleanFiles and directories specified
            // in cleanDirs.
            //!! Does this need afterEvaluate()?
            projectTasks.register("clean", Delete.class, task -> {
                task.setDescription("Clean up intermediate files.");
                task.setGroup(TASK_GROUP);

                ConfigurableFileTree tree = project.fileTree(projectDirectory);
                tree.include(lecture.getCleanFiles().get());
                task.delete(tree);

                lecture.getCleanDirs().get().forEach(
                        pattern -> task.delete(getListOfDirectories(projectDirectory, pattern))
                );
            });

            // Remove everything not under version control. Unfortunately this *doesn't*
            // include the .gradle directory :(, but in hindsight that's not surprising.
            projectTasks.register("deepClean", Exec.class, task -> {
                task.setDescription("Remove all files not in Git.");
                task.setGroup(TASK_GROUP);
                task.setCommandLine("git", "clean", "-fXd");
            });

            /**
             * Per-file tasks to convert images.
             *
             * By default these assume single-file inputs, e.g., foo.svg ->
             * foo.pdf. If you have a file that is dependent on multiple inputs
             * (e.g., bar.pu -> foo.pu -> foo.pdf), register an independent task
             * in the lecture's build.gradle.kts by copying the corresponding
             * registration code below, substituting the actual filenames in
             * place of the variable references like key and
             * generatedImages[key]. Also remove the file from generatedImages
             * to avoid a name clash. You can access properties like imageDir
             * via project.properties (these will probably need explicit type
             * casting).
             */
            // Generate per-file tasks to convert PlantUML to PDF.
            registerGeneratedImageTask(
                    ".*\\.(pu|puml|plantuml)$",
                    ".*\\.pdf$",
                    // Direct PDF from PlantUML is problematic, go via SVG instead.
                    new ArrayList<>(Arrays.asList(
                            "bash", "-c",
                            "plantuml -tsvg -pipe < <<src>> | rsvg-convert --format pdf > <<dst>> && pdfcrop <<dst>> <<cropped>> && mv -f <<cropped>> <<dst>>"
                    ))
            );

            // Generate per-file tasks to convert SVG to PDF.
            registerGeneratedImageTask(
                    ".*\\.svg$",
                    ".*\\.pdf$",
                    new ArrayList<>(Arrays.asList(
                            "bash", "-c",
                            "rsvg-convert --format pdf <<src>> > <<dst>> && pdfcrop <<dst>> <<cropped>> && mv -f <<cropped>> <<dst>>"
                    ))
            );

            // Generate per-file tasks to convert R to PDF.
            registerGeneratedImageTask(
                    ".*\\.R$",
                    ".*\\.pdf$",
                    new ArrayList<>(Arrays.asList(
                            "bash", "-c",
                            "R --no-echo --file=<<src>> --args <<dst>> && pdfcrop <<dst>> <<cropped>> && mv -f <<cropped>> <<dst>>"
                    ))
            );
        });

        // Debugging information.
        projectTasks.register("debug", task -> {
            task.setDescription("Print debugging information.");
            task.setGroup(TASK_GROUP);
            task.doLast(s -> {
                System.out.println("project name = <" + project.getName() + ">");
                System.out.println("project directory = <" + projectDirectory + ">");
                System.out.println("docTargets = <" + lecture.getDocTargets().get().toString() + ">");
                System.out.println("numberedDocTargets = <" + numberedDocTargets.get().toString() + ">");
                System.out.println("docPrefix = <" + lecture.getDocPrefix().get() + ">");
                System.out.println("lectureNum = <" + lecture.getLectureNum().get() + ">");
                System.out.println("docNames = <" + docNames.get().toString() + ">");
                System.out.println("numberedDocNames = <" + numberedDocNames.get().toString() + ">");
                System.out.println("imageDir = <" + lecture.getImageDir().get().getAsFile().getName() + ">");
                System.out.println("pdfDir = <" + lecture.getPdfDir().get().getAsFile().getName() + ">");
                System.out.println("slidesImages = <" + lecture.getSlidesImages().get().toString() + ">");
                System.out.println("slidesFiles = <" + lecture.getSlidesFiles().get().toString() + ">");
                System.out.println("examplesImages = <" + lecture.getExamplesImages().get().toString() + ">");
                System.out.println("examplesFiles = <" + lecture.getExamplesFiles().get().toString() + ">");
                System.out.println("slideDependencies = <" + slideDependencies.get().toString() + ">");
                System.out.println("docDependencies = <" + docDependencies.get().toString() + ">");
                System.out.println("generatedImages = <" + lecture.getGeneratedImages().get().toString() + ">");
                System.out.println("cleanFiles = <" + lecture.getCleanFiles().get().toString() + ">");
                System.out.println("cleanDirs = <" + lecture.getCleanDirs().get().toString() + ">");
                System.out.println("latexFlags = <" + lecture.getLatexFlags().get().toString() + ">");
                System.out.println("latex.terminalEmulator = <" + latex.getTerminalEmulator().get() + ">");
                System.out.println("latex.waitTime = <" + latex.getWaitTime().get() + ">");
                System.out.println("latex.waitUnit = <" + latex.getWaitUnit().get() + ">");
                System.out.println("latex.pdfLatexCommand = <" + latex.getPdfLatexCommand().get() + ">");
                System.out.println("latex.bibTexCommand = <" + latex.getBibTexCommand().get() + ">");
                System.out.println("latex.inkscapeCommand = <" + latex.getInkscapeCommand().get() + ">");
                System.out.println("latex.gitLatexdiffCommand = <" + latex.getGitLatexdiffCommand().get() + ">");
            });
        });
    }
}