User:GreenC bot/Job 18/source

#!/usr/local/bin/gawk -bE 
#
# vebug - https://en.wikipedia.org/wiki/User:GreenC_bot/Job_18
#
# The MIT License (MIT)
# 
# Copyright (c) August 2019 User:GreenC (en.wikipedia.org)
# 
# Permission is hereby granted, free of charge, to any person obtaining a copy
# of this software and associated documentation files (the "Software"), to deal
# in the Software without restriction, including without limitation the rights
# to use, copy, modify, merge, publish, distribute, sublicense, and/or sell
# copies of the Software, and to permit persons to whom the Software is
# furnished to do so, subject to the following conditions:
#
# The above copyright notice and this permission notice shall be included in
# all copies or substantial portions of the Software.
#
# THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR 
# IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
# FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE
# AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER 
# LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM,
# OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN
# THE SOFTWARE.
BEGIN {
 BotName = "vebug"
}
@include "botwiki.awk"
@include "library.awk"
@include "json.awk"
BEGIN {
 Mode = "bot" # set to "find" and it will search only and exit with a 1 (found something) or 0 (found nothing)
 # in "find" mode, run via 'project -s' to search local cache for articles containing actionable matches
 # set to anything else and it will process the article.
 IGNORECASE = 1
 ReSpace = "[\n\r\t]*[ ]*[\n\r\t]*[ ]*[\n\r\t]*"
 ReSups = "[<]sup[^>]*[>][^<]+[<][/]sup[>]"
 delete citeTable
 Optind = Opterr = 1
 while ((C = getopt(ARGC, ARGV, "hs:l:n:")) != -1) {
 opts++
 if(C == "s") # -s <file> article.txt source to process.
 articlename = verifyval(Optarg)
 if(C == "l") # -l <dir/> Directory where logging is sent.. end with "/"
 logdir = verifyval(Optarg)
 if(C == "n") # -n <name> Wikipedia name of article
 wikiname = verifyval(Optarg)
 if(C == "h") {
 usage()
 exit
 }
 }
 if( ! opts || articlename == "" ) {
 stdErr("Error in vebug.awk (1)")
 print "0"
 exit
 }
 if(wikiname == "" || logdir == "")
 Logfile = "/dev/null"
 else {
 if(substr(logdir, length(logdir), 1) != "/")
 logdir = logdir "/"
 Logfile = logdir "logvebug"
 }
 # Path to data directory ends in "/"
 DataDir = articlename
 gsub(regesc3(basename(articlename)) "$", "", DataDir) 
 # Number of changes made to article
 Count = 0
 main()
}
#
# Run the program, save the page to disk
#
function main( article,articlenew,articlenewname,editsummaryname,bn,plural) {
 checkexists(articlename, "vebug.awk main()", "exit")
 article = readfile(articlename)
 if(length(article) < 10) {
 print "0"
 exit
 }
 article = deflate(article)
 articlenew = vebug(article)
 if(article != articlenew && length(articlenew) > 10 && Count > 0) {
 articlenew = inflate(articlenew)
 articlenewname = DataDir "article.vebug.txt"
 printf("%s", articlenew) > articlenewname 
 close(articlenewname)
 editsummaryname = DataDir "editsummary.vebug.txt"
 if(Count > 1)
 plural = "s"
 printf("Restore %s cite" plural " deleted by a bug in VisualEditor ([[User:GreenC_bot/Job_18|vebug bot]])", Count) > editsummaryname # Customize the edit summary to be more specific
 close(editsummaryname)
 print Count
 exit
 }
 print "0"
 exit
}
#
# vebug - parse page, load data into citeTable[][], modify page, return it to main()
#
function vebug(article, c,field,sep,i,field2,sep2,j,d,k,dest,dest2,fieldi,field2txt,setdone,key,vertemp,re,foundit) {
 # Special case
 # Regularize <sup>[[User:Claudia Diaz2/sandbox#cite%20note-5|[6]]<nowiki>]</nowiki></sup>
 c = patsplit(article, field, /[|][[][^]]*[]]{2}[<]nowiki[>][]][<][/]nowiki[>]/, sep) 
 for(k = 1; k <= c; k++) {
 if(match(reverse(sep[k-1]), /[^[]+[[]{2}/, dest) ) {
 origcite = field[k]
 gsub(/<\/?nowiki>/, "", field[k])
 field[k] = reverse(dest[0]) field[k] 
 origTable[gsubi("^[<]nowiki[^>]*[>]|[<][/]nowiki[>]", "", field[k])] = reverse(dest[0]) origcite
 sep[k-1] = gsubs(reverse(dest[0]), "", sep[k-1])
 }
 }
 article = unpatsplit(field, sep)
 # Special case
 # Regularize [[Politics of Venezuela#cite%20note-19|<sup>[1</sup>]]
 c = patsplit(article, field, /[|][<]sup[>][[][0-9]*[<][/]sup[>][]]{2}/, sep) 
 for(k = 1; k <= c; k++) {
 if(match(reverse(sep[k-1]), /[^[]+[[]{2}/, dest) ) {
 origcite = field[k]
 sub(/[<]sup[>][[][^<]*[<][/]sup[>]/, "[]", field[k])
 field[k] = reverse(dest[0]) field[k] 
 origTable[gsubi("^[<]nowiki[^>]*[>]|[<][/]nowiki[>]", "", field[k])] = reverse(dest[0]) origcite
 sep[k-1] = gsubs(reverse(dest[0]), "", sep[k-1])
 }
 }
 article = unpatsplit(field, sep)
 # Special case
 # Regularize <sup>[[User:Claudia Diaz2/sandbox#cite%20note-5|<nowiki>6]</nowiki>]]</sup>
 c = patsplit(article, field, /[|][<]nowiki[>][^]]*[]]{1}[<][/]nowiki[>][]]{2}/, sep) 
 for(k = 1; k <= c; k++) {
 if(match(reverse(sep[k-1]), /[^[]+[[]{2}/, dest) ) {
 origcite = field[k]
 sub(/<nowiki>[^]]*[^]]/, "[", field[k])
 sub(/<\/nowiki>/, "", field[k])
 field[k] = reverse(dest[0]) field[k] 
 origTable[gsubi("^[<]nowiki[^>]*[>]|[<][/]nowiki[>]", "", field[k])] = reverse(dest[0]) origcite
 sep[k-1] = gsubs(reverse(dest[0]), "", sep[k-1])
 }
 }
 article = unpatsplit(field, sep)
 # Special case
 # Regularize [[2017 Women's March#cite%20note-FT%20100%2C000-13|<span>[13]</span>]]
 # works for <sup> or <span>
 split("sup span", tag, " ")
 for(k = 1; k <= 2; k++) {
 re = "[<]" tag[k] "[^>]*[>][[][0-9]+[]][<][/]" tag[k] "[>][]]{2}"
 c = patsplit(article, field, re, sep)
 for(i = 1; i <= c; i++) {
 if(match(reverse(sep[i-1]), /[|][^[]+[[]{2}/, dest) ) { 
 origcite = field[i] 
 re = "^[<]" tag[k] "[^>]*[>]|[<][/]" tag[k] "[>]" 
 gsub(re, "", field[i]) 
 field[i] = "<sup>" reverse(dest[0]) field[i] "</sup>" 
 origTable[gsubi("^[<]" tag[k] "[^>]*[>]|[<][/]" tag[k] "[>]","",field[i])] = reverse(dest[0]) origcite 
 sep[i-1] = gsubs(reverse(dest[0]), "", sep[i-1]) 
 }
 }
 article = unpatsplit(field,sep)
 }
 # Special case
 # Convert cases not surrounded by <sup> or <span> 
 d = patsplit(article, field2, ReSups, sep2) # Everything already surrounded by sup
 c = patsplit(reverse(article), field, /[]]{3}[0-9]{0,3}[[][|][0-9]{1,3}[-][^[]+[[]{2}/, sep)
 for(i = 1; i <= c; i++) {
 foundit = 0
 for(j = 1; j <= d; j++) {
 if(field2[j] ~ regesc3(reverse(field[i]))) {
 foundit = 1
 }
 }
 if( ! foundit) {
 origTable["<sup>" reverse(field[i]) "</sup>"] = reverse(field[i])
 field[i] = ">pus/<" field[i] ">pus<"
 }
 }
 article = reverse(unpatsplit(field, sep))
 # Standard: convert cites surrounded by <sup>..</sup>
 c = patsplit(article, field, ReSups, sep)
 for(i = 1; i <= c; i++) {
 if(field[i] !~ /[{][{][^{]*[{][{]/ && field[i] ~ /cite[%]20/) { # skip embeded templates not found by deflate()
 sendlog(logdir "logsups", wikiname, field[i])
 # Encode embedded [0-9] so it can be parsed
 # <sup>[[Group of Eight#cite%20note-19|[19]]][[Group of Eight#cite%20note-20|[20]]]</sup> -->
 # <sup>[[Group of Eight#cite%20note-19|VEBUGO19VEBUGC]][[Group of Eight#cite%20note-20|VEBUGO20VEBUGC]]</sup>
 fieldi = field[i]
 while(match(fieldi, "[[][0-9]+[]]", dest)) {
 if(match(dest[0], /[0-9]+/, dest2)) 
 field[i] = gsubs(dest[0], "VEBUGO" dest2[0] "VEBUGC", field[i])
 sub("[[][0-9]+[]]", "", fieldi)
 }
 # Populate citeTable[][]
 delete citeTable
 d = patsplit(field[i], field2, /[[]{2}[^]]+[]]{1,3}/, sep2)
 for(j = 1; j <= d; j++) {
 # Decoded
 field2txt = field2[j]
 field2txt = gsubs("VEBUGO", "[", field2txt)
 field2txt = gsubs("VEBUGC", "]", field2txt)
 citeTable[field2txt]["decoded"] = field2txt
 key = field2txt
 # Encoded
 citeTable[key]["encoded"] = field2[j]
 if( empty(origTable[key]))
 origTable[key] = key
 # Cite number
 getCiteNumbers(key)
 if(abort(key, "citenumber")) continue
 # Primary article
 getTitle(key, "artprimary")
 if(abort(key, "artprimary")) continue
 # Secondary article title eg. Group of Eight 
 getTitle(key, "artsecondary")
 if(abort(key, "artsecondary")) continue
 # Time/revision it was last added to primary article
 getPrimaryRevTime(key)
 if(abort(key, "artprimaryrevid")) continue
 if(abort(key, "artprimarytimestamp")) continue
 # Time/revision it existed in secondary article
 getSecondaryRevTime(key)
 if(abort(key, "artsecondaryrevid")) continue
 if(abort(key, "artsecondarytimestamp")) continue
 if(abort(key, "artsecondarywikitext")) continue
 setdone = 0 # if 1, ref is established early in process due to missing data
 settype = 0
 if(! empty(citeTable[key]["citenumbermismatch"])) {
 split(citeTable[key]["citenumbermismatch"], cites, /[|]/)
 vertemp = "{{verify source |date=" getDateToday() " |reason=This ref was deleted Special:Diff/" citeTable[key]["artprimaryrevid"] " by a bug in VisualEditor and later identified by a bot. The original cite can be found at Special:Permalink/" citeTable[key]["artsecondaryrevid"] " (or in a rev close to it) in either cite #" cites[1] " or cite #" cites[2] " - find and verify the cite and replace this template with it (1). [[User:GreenC_bot/Job_18]]}}"
 field2[j] = "<ref>Citation error. See inline comment how to fix. " vertemp "</ref>"
 settype = 1
 setdone = 1
 }
 # Get cite from secondary by its number
 if( ! setdone) {
 getSecondaryCiteByNumer(key)
 if( empty(citeTable[key]["citesecondaryplaintext"]) ) {
 vertemp = "{{verify source |date=" getDateToday() " |reason=This ref was deleted Special:Diff/" citeTable[key]["artprimaryrevid"] " by a bug in VisualEditor and later identified by a bot. The original cite can be found at Special:Permalink/" citeTable[key]["artsecondaryrevid"] " (or in a rev close to it) as cite #" citeTable[key]["citenumber"] " - find and verify the cite and replace this template with it (2). [[User:GreenC_bot/Job_18]]}}"
 field2[j] = "<ref>Citation error. See inline comment how to fix. " vertemp "</ref>"
 settype = 2
 setdone = 1
 }
 }
 # Upload page to wikipedia
 if( ! setdone) {
 if( generateCitationsPage(key) == 0) {
 vertemp = "{{verify source |date=" getDateToday() " |reason=This ref was deleted Special:Diff/" citeTable[key]["artprimaryrevid"] " by a bug in VisualEditor and later restored by a bot in plain-text form. The original cite can be found at Special:Permalink/" citeTable[key]["artsecondaryrevid"] " (or in a rev close to it) as cite #" citeTable[key]["citenumber"] " - find and verify the cite and replace this template with it (3). [[User:GreenC_bot/Job_18]]}}"
 field2[j] = "<ref>" citeTable[key]["citesecondaryplaintext"] " " vertemp "</ref>"
 settype = 3
 setdone = 1
 }
 }
 # Parse page
 if( ! setdone) {
 parseCitationsPage(key)
 if( empty(citeTable[key]["citesecondarywikicite"]) ) { 
 vertemp = "{{verify source |date=" getDateToday() " |reason=This ref was deleted Special:Diff/" citeTable[key]["artprimaryrevid"] " by a bug in VisualEditor and later restored by a bot in plain-text form. The original cite can be found at Special:Permalink/" citeTable[key]["artsecondaryrevid"] " (or in a rev close it it) as cite #" citeTable[key]["citenumber"] " - find and verify the cite and replace this template with it (4). [[User:GreenC_bot/Job_18]]}}"
 field2[j] = "<ref>" citeTable[key]["citesecondaryplaintext"] " " vertemp "</ref>"
 settype = 4
 setdone = 1
 }
 }
 if( ! setdone) {
 vertemp = "{{verify source |date=" getDateToday() " |reason=This ref was deleted Special:Diff/" citeTable[key]["artprimaryrevid"] " by a bug in VisualEditor and later restored by a bot from the original cite located at Special:Permalink/" citeTable[key]["artsecondaryrevid"] " cite #" citeTable[key]["citenumber"] " - verify the cite is accurate and delete this template. [[User:GreenC_bot/Job_18]]}}"
 field2[j] = "<ref>" citeTable[key]["citesecondarywikicite"] " " vertemp "</ref>"
 sendlog(logdir "logconvert", wikiname, key " ---- " citeTable[key]["citesecondarywikicite"] )
 }
 else {
 sendlog(logdir "logconvert", wikiname, key " ---- " settype )
 }
 Count++
 
 }
 field[i] = unpatsplit(field2,sep2)
 gsub(/^[<]sup[^>]*[>]|[<][/]sup[>]$/, "", field[i])
 # Save debuggung info to table.out in data directory
 saveTable()
 }
 } 
 article = unpatsplit(field, sep)
 # Check for missed sups 
 if(article ~ /[#]cite[%]20/ || article ~ /VEBUG[OC]/ ) {
 sendlog(logdir "logmissed", wikiname, "Contains cite%20 or VEBUGO . aborting")
 print article > DataDir "article.abort.txt"
 Count = 0 # Abort changes to article
 }
 return article
}
#
# Get citations numbers
#
# Populates:
# citeTable[key]["citenumber"] (a single cite number best guess, usually the second number)
# citeTable[key]["citenumbermismatch"] (two cites numbers sep by "|" if different, otherwise blank)
#
function getCiteNumbers(key, buf1,buf2,dest) {
 citeTable[key]["citenumber"] = ""
 citeTable[key]["citenumbermismatch"] = ""
 # -55|[note 1]]$ (get "note 1")
 if(match(reverse(citeTable[key]["decoded"]), /^[]]{2}[^[]*[[][|]/, dest)) {
 gsub(/^[]]{2,3}|[[][|]$/, "", dest[0])
 buf1 = reverse(dest[0])
 if(! empty(strip(buf1)))
 citeTable[key]["citenumber"] = buf1
 }
 # -55|[note 1]]$ (get "55")
 if(match(reverse(citeTable[key]["decoded"]), /[|][^-]*[^-]/, dest)) {
 gsub(/^[|]/, "", dest[0])
 if(! empty(strip(dest[0])))
 buf2 = reverse(dest[0])
 }
 if(! empty(citeTable[key]["citenumber"]) && ! empty(buf2)) {
 if(buf2 != citeTable[key]["citenumber"])
 citeTable[key]["citenumbermismatch"] = buf2 "|" citeTable[key]["citenumber"] 
 }
 if( ! isanumber(citeTable[key]["citenumber"]) && isanumber(buf2))
 citeTable[key]["citenumber"] = buf2
}
#
# Parse citations page User:GreenC/testcases/iabdebug
#
# Populate citeTable[key]["citesecondarywikicite"]
#
function parseCitationsPage(key, begin,i,a,b,fp,np,c,d,e) {
 citeTable[key]["citesecondarywikicite"] = ""
 # Convert page to plain-text
 command = "w3m -dump -cols 10000 'https://en.wikipedia.org/wiki/User:GreenC/testcases/iabdebug'"
 fp = sys2var(command)
 # Extract core surrounded by "%%%%" and "^^^^"
 begin = 0
 for(i = 1; i <= splitn(fp, a, i); i++) {
 if(a[i] ~ /^[%]{4}$/) { 
 begin = 1
 continue
 }
 if(a[i] ~ /^[\\^]{4}$/) begin = 0
 if(begin) 
 np = np "\n" a[i]
 }
 # Extract records surrounded by "@@@@"
 c = split(np, b, /[@]{4}\n/)
 for(i = 1; i <= c; i++) {
 if(! empty(strip(b[i]))) {
 d = split(strip(b[i]), e, /[+]{4}\n/)
 e[1] = gsubs("^[dead link]", "", e[1])
 # print strip(e[1]) " = " citeTable[key]["citesecondaryplaintext"]
 if(strip(e[1]) == citeTable[key]["citesecondaryplaintext"]) {
 citeTable[key]["citesecondarywikicite"] = strip(e[2])
 break
 }
 }
 }
 
}
#
# Generate a list of citations in parsable format and upload to User:GreenC/testcases/iabdebug
#
function generateCitationsPage(key, c,b,i,k,bp,np,status) {
 bp = "\n<p>\n"
 np = setdatetype(citeTable[key]["artsecondarywikitext"]) bp # need to set date type so CS1|2 date display is consistent
 np = np "%%%%" bp
 c = split(citeTable[key]["artsecondarywikitext"], b, "<ref[^>]*>")
 for(i = 1; i <= c; i++) {
 k = strip(substr(b[i], 1, match(b[i], "</ref>") - 1))
 if( ! empty(k)) {
 np = np "@@@@" bp
 np = np k bp
 np = np "++++" bp
 np = np "<nowiki>" k "</nowiki>" bp
 }
 } 
 np = np "^^^^" bp 
 # Upload page
 for(i = 1; i <= 3; i++) {
 status = sys2varPipe(np, "wikiget -E " shquote("User:GreenC/testcases/iabdebug") " -S " shquote(citeTable[key]["artprimary"] " -/- " citeTable[key]["artsecondary"]) " -P STDIN") 
 if(status ~ "Success" || status ~ "No change") {
 sleep(5)
 return 1
 }
 else
 sleep(5)
 }
 sendlog(logdir "syslog", wikiname, "Error: Unable to upload to User page, wikiget returns: " status)
 return 0
 
}
#
# Get plain-text version of given cite number in secondary article
# populates citeTable[key]["citesecondaryplaintext"]
# on error set to blank
#
function getSecondaryCiteByNumer(key, command,fp,citenum,a,i) {
 citeTable[key]["citesecondaryplaintext"] = ""
 # Plain text of secondary article
 command = "w3m -dump -cols 10000 " shquote("https://en.wikipedia.org/w/index.php?title=" urlencodeawk(citeTable[key]["artsecondary"]) "&oldid=" citeTable[key]["artsecondaryrevid"] )
 fp = sys2var(command)
 # Get the cite # in plain-text
 if( int(citeTable[key]["citenumber"]) < 10)
 citenum = "^[ ]" citeTable[key]["citenumber"] "[.][ ][\\^]"
 else
 citenum = "^" citeTable[key]["citenumber"] "[.][ ][\\^]"
 for(i = 1; i <= splitn(fp, a, i); i++) {
 if(a[i] ~ citenum) {
 sub(citenum, "", a[i])
 gsub(/[\\^][a-z]{1,3}[ ]/, "", a[i])
 a[i] = gsubs("^[dead link]", "", a[i])
 citeTable[key]["citesecondaryplaintext"] = strip(a[i])
 # break # keep going to get past any duplicates in 'notes' section .. this is imperfect though
 }
 }
}
#
# What time and revision was the bug added to the primary article? 
# populates:
# citeTable[key]["artsecondaryrevid"] 
# citeTable[key]["artsecondarytimestamp"]
# citeTable[key]["artsecondarywikitext"]
# on error they are blank 
# 
function getSecondaryRevTime(key, jsona,i,cont,unixprimary,command,j,arrevid,artimestamp,arcontinue,a,maxrevs,prevcontinue) {
 citeTable[key]["artsecondaryrevid"] = ""
 citeTable[key]["artsecondarytimestamp"] = ""
 citeTable[key]["artsecondarywikitext"] = ""
 if(! empty(citeTable[key]["artprimarytimestamp"]))
 unixPrimary = unixTime(citeTable[key]["artprimarytimestamp"])
 else
 return
 maxrevs = 20
 i = 0
 cont = 1
 while(cont) {
 i++
 if(i > maxrevs) { 
 sendlog(logdir "syslog", wikiname, "Error: Exceeded " maxrevs " API requests in getSecondaryRevTime()")
 break # sanity break
 }
 if(empty(arcontinue["continue"]))
 command = "wget -q -O- " shquote("https://en.wikipedia.org/w/api.php?action=query&prop=revisions&titles=" urlencodeawk(citeTable[key]["artsecondary"]) "&rvlimit=50&rvslots=main&rvprop=timestamp%7Cuser%7Cids&rvdir=older&format=json") 
 else
 command = "wget -q -O- " shquote("https://en.wikipedia.org/w/api.php?action=query&prop=revisions&titles=" urlencodeawk(citeTable[key]["artsecondary"]) "&rvlimit=50&rvslots=main&rvprop=timestamp%7Cuser%7Cids&rvdir=older&format=json&rvcontinue=" urlencodeawk(arcontinue["continue"]) ) 
 if(query_json(sys2var(command), jsona) >= 0) {
 #debug
 #awkenough_dump(jsona, "jsona")
 # jsona["query","pages","1196634","revisions","1","revid"]=7741763
 splitja(jsona, arrevid, 5, "revid")
 # jsona["query","pages","1196634","revisions","1","timestamp"]=2004年11月22日T05:21:18Z
 splitja(jsona, artimestamp, 5, "timestamp")
 # jsona["continue","rvcontinue"]=20041122055516|7769047
 splitja(jsona, arcontinue, 1, "rvcontinue")
 for(j = 1; j <= length(arrevid); j++) {
 if(unixTime(artimestamp[j]) <= unixPrimary) {
 if(unixTime(artimestamp[j]) == unixPrimary && citeTable[key]["artprimary"] == citeTable[key]["artsecondary"]) { # same article same diff get prev
 if(j == 50) break # size of block requested in API
 j++
 }
 delete a
 tup(getwikisource(citeTable[key]["artsecondary"], "dontfollow", "wikipedia.org", "en", arrevid[j]), a)
 if(a[1] != "REDIRECT" && ! empty(a[1])) {
 citeTable[key]["artsecondaryrevid"] = arrevid[j]
 citeTable[key]["artsecondarytimestamp"] = artimestamp[j]
 citeTable[key]["artsecondarywikitext"] = a[1]
 cont = 0
 break
 }
 else {
 if( empty(a[1]) && empty(a[2]) ) { # revision is empty
 sendlog(logdir "syslog", wikiname, "Error: empty secondary revision (" citeTable[key]["artsecondary"] "): " artimestamp[j])
 }
 cont = 0
 break
 }
 }
 }
 prevcontinue = arcontinue["continue"]
 }
 }
}
#
# Get article title (follow redirects)
#
function getTitle(key1,key2, dest,a) {
 citeTable[key1][key2] = ""
 if(key2 == "artsecondary") {
 if(match(key1, /^[[][^#]+[^#]/, dest) > 0) {
 sub(/^[[]{2}/, "", dest[0])
 tup(getwikisource(dest[0], "dontfollow", "wikipedia.org", "en"), a)
 if(a[1] == "REDIRECT") {
 gsub(/^#REDIRECT[ ]*[[]{2}|[]]$/, "", a[2])
 citeTable[key1][key2] = a[2]
 }
 else
 citeTable[key1][key2] = dest[0]
 }
 }
 else if(key2 = "artprimary") {
 tup(getwikisource(wikiname, "dontfollow", "wikipedia.org", "en"), a)
 if(a[1] == "REDIRECT") {
 gsub(/^#REDIRECT [[]{2}|[]]$/, "", a[2])
 citeTable[key1][key2] = a[2]
 }
 else
 citeTable[key1][key2] = wikiname
 }
}
#
# What time and revision was the bug added to the primary article? 
# populates:
# citeTable[key]["artprimaryrevid"] 
# citeTable[key]["artprimarytimestamp"]
# on error they are blank 
# 
function getPrimaryRevTime(key, jsona,i,cont,command,arrevid,atimestamp,arcontinue,j,a,prevrevid,prevtimestamp,maxrevs,prevcontinue,jsonin) {
 citeTable[key]["artprimaryrevid"] = ""
 citeTable[key]["artprimarytimestamp"] = ""
 maxrevs = 20
 i = 0
 cont = 1
 while(cont) {
 i++
 if(i > maxrevs) { 
 sendlog(logdir "syslog", wikiname, "Error: Exceeded " maxrevs " API requests in getPrimaryRevTime()")
 break # sanity break
 }
 if(empty(arcontinue["continue"]))
 command = "wget -q -O- " shquote("https://en.wikipedia.org/w/api.php?action=query&prop=revisions&titles=" urlencodeawk(citeTable[key]["artprimary"]) "&rvlimit=50&rvslots=main&rvprop=timestamp%7Cuser%7Cids&rvdir=older&format=json") 
 else
 command = "wget -q -O- " shquote("https://en.wikipedia.org/w/api.php?action=query&prop=revisions&titles=" urlencodeawk(citeTable[key]["artprimary"]) "&rvlimit=50&rvslots=main&rvprop=timestamp%7Cuser%7Cids&rvdir=older&format=json&rvcontinue=" urlencodeawk(arcontinue["continue"]) ) 
 jsonin = sys2var(command)
 if(query_json(jsonin, jsona) >= 0) {
 #debug
 #awkenough_dump(jsona, "jsona")
 # jsona["query","pages","1196634","revisions","1","revid"]=7741763
 splitja(jsona, arrevid, 5, "revid")
 # jsona["query","pages","1196634","revisions","1","timestamp"]=2004年11月22日T05:21:18Z
 splitja(jsona, artimestamp, 5, "timestamp")
 # jsona["continue","rvcontinue"]=20041122055516|7769047
 splitja(jsona, arcontinue, 1, "rvcontinue")
 if(arcontinue["continue"] == prevcontinue) { # reached last revision
 j = length(arrevid) 
 delete a
 tup(getwikisource(citeTable[key]["artprimary"], "dontfollow", "wikipedia.org", "en", arrevid[j]), a)
 if(a[1] != "REDIRECT" && ! empty(a[1])) {
 citeTable[key]["artprimaryrevid"] = arrevid[j]
 citeTable[key]["artprimarytimestamp"] = artimestamp[j]
 citeTable[key]["artprimarywikitext"] = a[1]
 cont = 0
 break
 }
 else {
 sendlog(logdir "syslog", wikiname, "Error: empty primary revision: " artimestamp[j])
 cont = 0
 break
 }
 }
 for(j = 1; j <= length(arrevid); j++) { # step through each revision for this batch
 tup(getwikisource(citeTable[key]["artprimary"], "dontfollow", "wikipedia.org", "en", arrevid[j]), a)
 if(a[1] != "REDIRECT" && ! empty(a[1])) {
 # if(countsubstring(a[1], citeTable[key]["decoded"]) == 0) {
 if(countsubstring(a[1], origTable[key]) == 0) {
 if(! empty(prevrevid)) {
 citeTable[key]["artprimaryrevid"] = prevrevid
 citeTable[key]["artprimarytimestamp"] = prevtimestamp
 }
 else {
 citeTable[key]["artprimaryrevid"] = arrevid[j]
 citeTable[key]["artprimarytimestamp"] = artimestamp[j]
 }
 cont = 0
 break
 }
 else {
 prevrevid = arrevid[j]
 prevtimestamp = artimestamp[j]
 }
 }
 else {
 if( empty(a[1]) && empty(a[2]) ) { # revision is empty
 if( citeTable[key]["artprimary"] != citeTable[key]["secondary"] ) {
 citeTable[key]["artprimaryrevid"] = arrevid[j]
 citeTable[key]["artprimarytimestamp"] = artimestamp[j]
 }
 else 
 sendlog(logdir "syslog", wikiname, "Error: empty primary revision: " artimestamp[j])
 cont = 0
 break
 }
 }
 }
 prevcontinue = arcontinue["continue"]
 }
 }
}
#
# Given a Wikipedia datestring, return unix-time string (seconds since 1970)
#
function unixTime(s) {
 return sys2var("date --date=" shquote(s) " +%s")
}
#
# Return todays date as "August 2019"
#
function getDateToday() {
 return sys2var("date +\"%B %Y\"")
}
#
# Determine article date type {{set dmy dates}} etc
# imported from medilibrary.nim
#
function setdatetype(art, reDmy,reMdy,i,a) {
 reDmy = "[{]{2}" ReSpace "use" ReSpace "dmy" ReSpace "d?a?t?e?s?|[{]{2}" ReSpace "dmy" ReSpace "[|]|[{]{2}" ReSpace "dmy" ReSpace "[}]|[{]{2}" "use[ -]?dmy"
 reMdy = "[{]{2}" ReSpace "use" ReSpace "mdy" ReSpace "d?a?t?e?s?|[{]{2}" ReSpace "mdy" ReSpace "[|]|[{]{2}" ReSpace "mdy" ReSpace "[}]|[{]{2}" "use[ -]?mdy"
 for(i = 1; i <= splitn(art, a, i); i++) {
 if(a[i] ~ reDmy)
 return "{{dmy}}"
 if(a[i] ~ reMdy)
 return "{{mdy}}"
 }
 return ""
}
#
# Abort check
#
function abort(key1, key2) {
 if( empty( strip(citeTable[key1][key2]) ) ) {
 if(key2 == "citesecondaryplaintext") {
 sendlog(logdir "logabort", wikiname, key1 " ---- " citeTable[key1]["citesecondaryplaintext"] " ---- " key2 " missing")
 }
 else {
 sendlog(logdir "logabort", wikiname, key1 " ---- " key2 " missing")
 }
 return 1
 }
 return 0
} 
#
# Reverse string
#
function reverse(s, i,len,a,r) {
 len = split(s, a, "")
 for(i = 1; i <= len; i++) 
 r = a[i] r
 return r
}
function saveTable() {
 # Save debuggung info to table.out in data directory
 printtable = 1
 if(printtable) {
 for(kk in citeTable) {
 print "\n -------------- \n" >> DataDir "table.out"
 print "origTable[" kk "] = " origTable[kk] >> DataDir "table.out"
 for(ll in citeTable[kk]) {
 if(ll != "artsecondarywikitext")
 print "citeTable[" kk "][" ll "] = " citeTable[kk][ll] >> DataDir "table.out"
 else {
 print "citeTable[" kk "][" ll "] = " length(citeTable[kk][ll]) >> DataDir "table.out"
 }
 }
 }
 }
}