2 description = "r/SneerClub's contents preserved as JSON files";
4 inputs = { flake-utils.url = "github:numtide/flake-utils"; };
6 outputs = { self, nixpkgs, flake-utils }:
7 flake-utils.lib.eachDefaultSystem (system:
9 pkgs = nixpkgs.legacyPackages."${system}";
10 process-markdown = pkgs.writeText "process-markdown.mlr" ''
11 subr process_replies(r) {
14 md = system("${pkgs.pandoc}/bin/pandoc -f markdown <<__EOF__\n".$r[i]["body"]."\n__EOF__");
15 utc = $r[i]["created_utc"];
19 $r[i]["created_date"] = system("date -d @".utc." -u +'%B %d, %Y %I:%M %p'");
21 call process_replies($r[i].replies);
24 $selftext = system("${pkgs.pandoc}/bin/pandoc -f markdown <<__EOF__\n".$selftext."\n__EOF__");
25 $created_date = system("date -d @".$created_utc." -u +'%B %d, %Y %I:%M %p'");
26 for (i, v in $comments) {
27 b = $comments[i].body;
28 md = system("${pkgs.pandoc}/bin/pandoc -f markdown <<__EOF__\n".b."\n__EOF__");
29 utc = $comments[i].created_utc;
31 $comments[i].body = md;
33 $comments[i].created_date = system("date -d @".utc." -u +'%B %d, %Y %I:%M %p'");
35 call process_replies($comments[i]["replies"]);
39 packages.json-threads = pkgs.runCommand "process-json" { } ''
42 tar -I ${pkgs.zstd}/bin/zstd -xf ${./bdfr.tar.zst}
43 ${pkgs.miller}/bin/mlr --json \
44 put -f ${process-markdown} then \
45 sort -nr created_utc \
46 ./bdfr/**/*.json > $out/threads-newest.json
47 ${pkgs.miller}/bin/mlr --json \
48 cut -x -f comments,selftext then \
50 $out/threads-newest.json > $out/submissions-bestest.json
51 ${pkgs.miller}/bin/mlr --json \
52 cut -x -f comments,selftext then \
53 sort -nr num_comments \
54 $out/threads-newest.json > $out/submissions-longest.json
57 packages.default = self.packages."${system}".json-threads;