Fetching all own statuses and bookmarks from a MastoAPI fediware

Table of Contents

1. But first, preparing a Nushell org-babel executor

  (require 'ob)

  (defvar org-babel-default-header-args:nushell
    '((:results . "output"))
    "Default header arguments for Nushell.")

  (defun org-babel-variable-assignments:nushell (params)  
  "Return list of Nushell statements assigning the block's variables."  
  (mapcar  
   (lambda (pair)  
     (format "let %s = %s"  
             (car pair)  
             (org-babel-nushell-var-to-nushell (cdr pair))))  
   (org-babel--get-vars params)))  

(defun org-babel-nushell-var-to-nushell (var)  
  "Convert an elisp value VAR to a Nushell variable."  
  ;; Implement conversion for strings, numbers, lists, etc.  
  (cond  
   ((stringp var) (format "\"%s\"" var))  
   ((numberp var) (format "%s" var))  
   ((listp var) (format "[%s]" (mapconcat #'org-babel-nushell-var-to-nushell var " ")))  
   (t (format "%s" var))))  

(defun org-babel-expand-body:nushell (body params)  
  "Expand BODY according to PARAMS, return the expanded body."  
  (let ((var-lines (org-babel-variable-assignments:nushell params)))  
    (org-babel-expand-body:generic body params var-lines)))

(defun org-babel-execute:nushell (body params)  
  "Execute Nushell BODY according to PARAMS."  
(let* ((expanded-body (org-babel-expand-body:nushell body params))  
       (tmpfile (org-babel-temp-file "nu-" ".nu"))  
       (cmd (or (cdr (assq :cmd params)) "nu"))  
       (result-params (cdr (assq :results params))))  
  (with-temp-file tmpfile  
    (insert expanded-body))  
  (unwind-protect  
      (org-babel-eval  
       (format "%s %s" cmd (shell-quote-argument tmpfile))  
       "")  
    (delete-file tmpfile))))

(add-to-list 'org-src-lang-modes '("nushell" . nushell))
(add-to-list 'org-babel-tangle-lang-exts '("nushell" . "nu"))

(provide 'ob-nushell)

;; (require 'ob-async) ;; couldn't get :async to work.

2. Preparing the access token

#+name: token
#+begin_src nushell :exports code :cache yes :results output verbatim
let username = ^kdialog --inputbox "name:"
let instance = ^kdialog --inputbox "instance:"
let password = ^kdialog --password "password:"
let client = (http post $"https://($instance)/api/v1/apps"
-t application/json
{
          client_name: "mastodon-archive"
          redirect_uris: "urn:ietf:wg:oauth:2.0:oob"
          scopes: "read write follow"
      })
let token = (http post $"https://($instance)/oauth/token"
-t application/json
($client | merge {
          grant_type: "password"
          username: $username
          password: $password
          scope: "read write follow"
      }))
print ($token | get access_token)

3. Running the fetches

#+begin_src nushell :exports code :var token=token
let instance = (open ~/.instance | str trim)
let bearer = { Authorization: $"Bearer ($token)" }
let api = $"https://($instance)/api/v1"
def api_get [url: string] {
  http get --headers $bearer -f $url
}
let acc = http get --headers $bearer $"($api)/accounts/verify_credentials"
def fetch_thing [dir, path: string] {
  mut url = $"($api)/($path)" # consider &limit=100
  loop {
    mut duplicates = 0
    let res = api_get $url
    for item in $res.body {
      let id = ($item | get id)
      let path = ($dir | path join $"($id).json")
      if ($path | path exists) {
        $duplicates = $duplicates + 1
      }
      $item | to json | save -f --raw $path
    }
    print $"($duplicates) dup in ($url)"

    # RFC 5988
    let raw_link_header = $res.headers.response | where name =~ '(?i)link'
    if ($raw_link_header | is-empty) {
      break
    }
    let link_header = ($raw_link_header | get 0.value)
    let parts = ($link_header | split row ",")
    let next_part = (
          $parts
          | where { $in | str contains 'rel="next"' }
          | first
      )
    $url = ($next_part | parse -r '<(?<url>[^>]+)>' | get 0.url)
  }
}
fetch_thing bookmarks bookmarks
fetch_thing statuses $"accounts/($acc.id)/statuses"

Unfortunately neither https://redlib.catsarch.com/r/emacs/comments/lx0dfd/orgbabel_and_long_running_processes/ ob-async nor ob-servant are easy to deal with and thus emacs simply hangs waiting for such a long-running process.

and the result looks like this (below I redacted the URLs):

20 dup in https:/api/v1/bookmarks
20 dup in https:/api/v1/bookmarks?max_id=262
20 dup in https:/api/v1/bookmarks?max_id=241
20 dup in https:/api/v1/bookmarks?max_id=221
20 dup in https:/api/v1/bookmarks?max_id=197
20 dup in https:/api/v1/bookmarks?max_id=175
20 dup in https:/api/v1/bookmarks?max_id=150
20 dup in https:/api/v1/bookmarks?max_id=126
20 dup in https:/api/v1/bookmarks?max_id=101
20 dup in https:/api/v1/bookmarks?max_id=76
20 dup in https:/api/v1/bookmarks?max_id=55
18 dup in https:/api/v1/bookmarks?max_id=31
0 dup in https://api/v1/bookmarks?max_id=8
20 dup in https:/api/v1/accounts8/statuses
20 dup in https:/api/v1/accounts8/statuses?max_id=B7q
20 dup in https:/api/v1/accounts8/statuses?max_id=B74
20 dup in https:/api/v1/accounts8/statuses?max_id=B70
. . .
0 dup in https://api/v1/accounts//statuses?max_id=
0 dup in https://api/v1/accounts//statuses?max_id=
0 dup in https://api/v1/accounts//statuses?max_id=
. . .

3.1. mastodon-archive

I couldn't use https://src.alexschroeder.ch/mastodon-archive.git/, which is otherwise pretty neat like with all the delta incremental fetching waiting for "Seen 10 duplicates, stopping now", due to

Version check failed (Need Mastodon instance version 3.1.0 to call this endpoint)

while fetching bookmarks.

It uses the Mastodon python library there so things like these are layered with the tar of abstraction.

Other pain points:

  • interruption meant all download result was lost
  • no way to monitor progress (here I can look at du --inodes)
  • no way to not echo the password on instances where I need to use it