Module:WikitextParser

 -- Module:WikitextParser is a general-purpose wikitext parser
 -- Documentation and master version: https://en.wikipedia.org/wiki/Module:WikitextParser
 -- Authors: User:Sophivorus, User:Certes, User:Aidan9382, et al.
 -- License: CC-BY-SA-4.0

 localstr_find,str_sub,str_match=string.find,string.sub,string.match
 localstr_gsub,str_gmatch,str_rep=string.gsub,string.gmatch,string.rep
 localstr_lower=string.lower
 localtable_insert,table_concat=table.insert,table.concat

 -- Private helper method to escape a string for use in regexes
 localfunctionescapeString(str)
 returnstr_gsub(str,'[%^%$%(%)%.%[%]%*%+%-%?%%]','%%%0')
 end

 -- Fast gsplit (a la mw.text.gsplit) for fixed ASCII separators
 localfunctionstr_gsplit(str,sep)
 ifsep==""orstr_find(sep,"[128円-255円]")then
 error("Separator must be ASCII and cannot be empty")
 end
 locali,str1=1,str..sep
 returnfunction()
 locals,e=str_find(str1,sep,i,true)
 ifnotsthen
 returns
 end
 localt=i
 i=e+1
 returnstr_sub(str1,t,s-1)
 end
 end

 -- Fast dedicated two-way split (a la mw.text.split) for fixed ASCII separators
 localfunctionstr_split2(str,sep)
 ifsep==""orstr_find(sep,"[128円-255円]")then
 error("Separator must be ASCII and cannot be empty")
 end
 locals,e=str_find(str,sep,1,true)
 ifnotsthen
 returnstr
 end
 returnstr_sub(str,1,s-1),str_sub(str,e+1)
 end

 -- Fast whitespace trim (a la mw.text.trim)
 localfunctionstr_trim(str)
 returnstr_match(str,'^%s*(.-)%s*$')
 end

 localparser={}

 -- Get the lead section from the given wikitext
 -- The lead section is any content before the first section title.
 -- @param wikitext Required. Wikitext to parse.
 -- @return Wikitext of the lead section. May be empty if the lead section is empty.
 functionparser.getLead(wikitext)
 wikitext='\n'..wikitext
 wikitext=str_gsub(wikitext,'\n==.*','')
 wikitext=str_trim(wikitext)
 returnwikitext
 end

 -- Get the sections from the given wikitext
 -- This method doesn't get the lead section, use getLead for that
 -- @param wikitext Required. Wikitext to parse.
 -- @return Map from section title to section content
 functionparser.getSections(wikitext)
 localsections={}
 wikitext='\n'..wikitext..'\n=='
 fortitleinstr_gmatch(wikitext,'\n==+ *([^=]-) *==+')do
 localsection=str_match(wikitext,'\n==+ *'..escapeString(title)..' *==+(.-)\n==')
 section=str_trim(section)
 sections[title]=section
 end
 returnsections
 end

 -- Get a section from the given wikitext (including any subsections)
 -- If the given section title appears more than once, only the section of the first instance will be returned
 -- @param wikitext Required. Wikitext to parse.
 -- @param title Required. Title of the section
 -- @return Wikitext of the section, or nil if it isn't found. May be empty if the section is empty or contains only subsections.
 functionparser.getSection(wikitext,title)
 title=str_trim(title)
 title=escapeString(title)
 wikitext='\n'..wikitext..'\n'
 locallevel
 level,wikitext=str_match(wikitext,'\n(==+) *'..title..' *==.-\n(.*)')
 ifwikitextthen
 localnextSection='\n=='..str_rep('=?',#level-2)..'[^=].*'
 wikitext=str_gsub(wikitext,nextSection,'')-- remove later sections at this level or higher
 wikitext=str_trim(wikitext)
 returnwikitext
 end
 end

 -- Get the content of a <section> tag from the given wikitext.
 -- We can't use getTags because unlike all other tags, both opening and closing <section> tags are self-closing.
 -- @param wikitext Required. Wikitext to parse.
 -- @param name Required. Name of the <section> tag
 -- @return Content of the <section> tag, or nil if it isn't found. May be empty if the section tag is empty.
 functionparser.getSectionTag(wikitext,name)
 name=str_trim(name)
 name=escapeString(name)
 localsections={}
 forsectioninstr_gmatch(wikitext,'< *section +begin *= *["\']? *'..name..' *["\']? */>(.-)< *section +end= *["\']? *'..name..' *["\']? */>')do
 table_insert(sections,section)
 end
 if#sections>0then
 returntable_concat(sections)
 end
 end

 -- Get the lists from the given wikitext.
 -- @param wikitext Required. Wikitext to parse.
 -- @return Sequence of lists.
 functionparser.getLists(wikitext)
 locallists={}
 wikitext='\n'..wikitext..'\n\n'
 forlistinstr_gmatch(wikitext,'\n([*#].-)\n[^*#]')do
 table_insert(lists,list)
 end
 returnlists
 end

 -- Get the paragraphs from the given wikitext.
 -- @param wikitext Required. Wikitext to parse.
 -- @return Sequence of paragraphs.
 functionparser.getParagraphs(wikitext)
 localparagraphs={}

 -- Remove non-paragraphs
 wikitext='\n'..wikitext..'\n'-- add newlines to simplfy patterns
 wikitext=str_gsub(wikitext,'%f[^\n]<!%-%-.-%-%->%f[\n]','')-- remove comments
 wikitext=str_gsub(wikitext,'%f[^\n]%[%b[]%]%f[\n]','')-- remove files and categories
 wikitext=str_gsub(wikitext,'%f[^\n]%b{} *%f[\n]','')-- remove tables and block templates
 wikitext=str_gsub(wikitext,'%f[^\n]%b{} *%b{} *%f[\n]','')-- remove neighboring tables and block templates
 wikitext=str_gsub(wikitext,'%f[^\n]%b{} *<!%-%-.-%-%-> *%b{} *%f[\n]','')-- remove neighboring tables and block templates with a comment among them
 wikitext=str_gsub(wikitext,'%f[^\n][*#].-%f[\n]','')-- remove lists
 wikitext=str_gsub(wikitext,'%f[^\n]==+[^=]+==+ *%f[\n]','')-- remove section titles
 wikitext=str_trim(wikitext)

 forparagraphinstr_gsplit(wikitext,"\n\n")do
 paragraph=str_trim(paragraph)
 ifparagraph~=""then-- Filter out 3+ consecutive newlines
 table_insert(paragraphs,paragraph)
 end
 end
 returnparagraphs
 end

 -- Get the templates from the given wikitext.
 -- @param wikitext Required. Wikitext to parse.
 -- @return Sequence of templates.
 functionparser.getTemplates(wikitext)
 localtemplates={}
 fortemplateinstr_gmatch(wikitext,'{%b{}}')do
 ifstr_sub(template,1,3)~='{{#'then-- skip parser functions like #if
 table_insert(templates,template)
 end
 end
 returntemplates
 end

 -- Get the requested template from the given wikitext.
 -- If the template appears more than once, only the first instance will be returned
 -- @param wikitext Required. Wikitext to parse.
 -- @param name Name of the template to get
 -- @return Wikitext of the template, or nil if it wasn't found
 functionparser.getTemplate(wikitext,name)
 localtemplates=parser.getTemplates(wikitext)
 locallang=mw.language.getContentLanguage()
 for_,templateinipairs(templates)do
 localtemplateName=parser.getTemplateName(template)
 iflang:ucfirst(templateName)==lang:ucfirst(name)then
 returntemplate
 end
 end
 end

 -- Get name of the template from the given template wikitext.
 -- @param templateWikitext Required. Wikitext of the template to parse.
 -- @return Name of the template
 -- @todo Strip "Template:" namespace?
 functionparser.getTemplateName(templateWikitext)
 returnstr_match(templateWikitext,'^{{ *([^}|\n]+)')
 end

 -- Get the parameters from the given template wikitext.
 -- @param templateWikitext Required. Wikitext of the template to parse.
 -- @return Map from parameter names to parameter values, NOT IN THE ORIGINAL ORDER.
 -- @return Order in which the parameters were parsed.
 functionparser.getTemplateParameters(templateWikitext)
 localparameters={}
 localparamOrder={}
 localparams=str_match(templateWikitext,'{{[^|}]-|(.*)}}')
 ifparamsthen
 -- Temporarily replace pipes in subtemplates and links to avoid chaos
 forsubtemplateinstr_gmatch(params,'{%b{}}')do
 params=str_gsub(params,escapeString(subtemplate),str_gsub(subtemplate,'.',{['%']='%%',['|']="@@:@@",['=']='@@_@@'}))
 end
 forlinkinstr_gmatch(params,'%[%b[]%]')do
 params=str_gsub(params,escapeString(link),str_gsub(link,'.',{['%']='%%',['|']='@@:@@',['=']='@@_@@'}))
 end
 localcount=0
 localname,value
 forparaminstr_gsplit(params,'|')do
 name,value=str_split2(param,'=')
 ifvaluethen
 localnumber=tonumber(name)
 ifnumberthen--XXX: What about keys like: 01, .1, 2e0, etc.
 name=number
 end
 value=str_trim(value)
 else
 value=name
 count=count+1
 name=count
 end
 value=str_gsub(value,'@@_@@','=')
 value=str_gsub(value,'@@:@@','|')
 parameters[name]=value
 table_insert(paramOrder,name)
 end
 end
 returnparameters,paramOrder
 end

 -- Get the tags from the given wikitext.
 -- @param wikitext Required. Wikitext to parse.
 -- @return Sequence of tags.
 functionparser.getTags(wikitext)
 localtags={}
 localtag,tagName,tagEnd
 -- Don't match closing tags like </div>, comments like <!--foo-->, comparisons like 1<2 or things like <3
 fortagStart,tagOpeninstr_gmatch(wikitext,'()(<[^/!%d].->)')do
 tagName=parser.getTagName(tagOpen)

 -- If we're in a self-closing tag, like <ref name="foo" />, <references/>, <br/>, <br>, <hr>, etc.
 ifstr_match(tagOpen,'<.-/>')ortagName=='br'ortagName=='hr'then
 tag=tagOpen

 -- If we're in a tag that may contain others like it, like <div> or <span>
 elseiftagName=='div'ortagName=='span'then
 localposition=tagStart+#tagOpen-1
 localdepth=1
 whiledepth>0do
 tagEnd=str_match(wikitext,'</ ?'..tagName..' ?>()',position)
 iftagEndthen
 tagEnd=tagEnd-1
 else
 break-- unclosed tag
 end
 position=str_match(wikitext,'()< ?'..tagName..'[ >]',position+1)
 ifnotpositionthen
 position=tagEnd+1
 end
 ifposition>tagEndthen
 depth=depth-1
 else
 depth=depth+1
 end
 end
 tag=str_sub(wikitext,tagStart,tagEnd)

 -- Else we're probably in tag that shouldn't contain others like it, like <math> or <strong>
 else
 tagEnd=str_match(wikitext,'</ ?'..tagName..' ?>()',tagStart)
 iftagEndthen
 tag=str_sub(wikitext,tagStart,tagEnd-1)

 -- If no end tag is found, assume we matched something that wasn't a tag, like <no. 1>
 else
 tag=nil
 end
 end
 table_insert(tags,tag)
 end
 returntags
 end

 -- Get the name of the tag in the given wikitext
 -- @param tag Required. Tag to parse.
 -- @return Name of the tag or nil if not found
 functionparser.getTagName(tagWikitext)
 localtagName=str_match(tagWikitext,'^< *(.-)[ />]')
 iftagNamethentagName=str_lower(tagName)end
 returntagName
 end

 -- Get the value of an attribute in the given tag.
 -- @param tagWikitext Required. Wikitext of the tag to parse.
 -- @param attribute Required. Name of the attribute.
 -- @return Value of the attribute or nil if not found
 functionparser.getTagAttribute(tagWikitext,attribute)
 local_,value=str_match(tagWikitext,'^<[^/>]*'..attribute..' *= *(["\']?)([^/>]-)%1[ />]')
 returnvalue
 end

 -- Get the content of the given tag.
 -- @param tagWikitext Required. Wikitext of the tag to parse.
 -- @return Content of the tag. May be empty if the tag is empty. Will be nil if the tag is self-closing.
 -- @todo May fail with nested tags
 functionparser.getTagContent(tagWikitext)
 returnstr_match(tagWikitext,'^<.->.-</.->')
 end

 -- Get the <gallery> tags from the given wikitext.
 -- @param wikitext Required. Wikitext to parse.
 -- @return Sequence of gallery tags.
 functionparser.getGalleries(wikitext)
 localgalleries={}
 localtags=parser.getTags(wikitext)
 for_,taginipairs(tags)do
 localtagName=parser.getTagName(tag)
 iftagName=='gallery'then
 table_insert(galleries,tag)
 end
 end
 returngalleries
 end

 -- Get the <ref> tags from the given wikitext.
 -- @param wikitext Required. Wikitext to parse.
 -- @return Sequence of ref tags.
 functionparser.getReferences(wikitext)
 localreferences={}
 localtags=parser.getTags(wikitext)
 for_,taginipairs(tags)do
 localtagName=parser.getTagName(tag)
 iftagName=='ref'then
 table_insert(references,tag)
 end
 end
 returnreferences
 end

 -- Get the reference with the given name from the given wikitext.
 -- @param wikitext Required. Wikitext to parse.
 -- @param referenceName Required. Name of the reference.
 -- @return Wikitext of the reference
 functionparser.getReference(wikitext,referenceName)
 localreferences=parser.getReferences(wikitext)
 for_,referenceinipairs(references)do
 localcontent=parser.getTagContent(reference)
 localname=parser.getTagAttribute(reference,'name')
 ifcontentandname==referenceNamethen
 returnreference
 end
 end
 end

 -- Get the tables from the given wikitext.
 -- @param wikitext Required. Wikitext to parse.
 -- @return Sequence of tables.
 functionparser.getTables(wikitext)
 localtables={}
 wikitext='\n'..wikitext
 fortinstr_gmatch(wikitext,'\n%b{}')do
 ifstr_sub(t,1,3)=='\n{|'then
 t=str_trim(t)-- exclude the leading newline
 table_insert(tables,t)
 end
 end
 returntables
 end

 -- Get the id from the given table wikitext
 -- @param tableWikitext Required. Wikitext of the table to parse.
 -- @param attribute Required. Name of the attribute.
 -- @return Value of the attribute or nil if not found
 functionparser.getTableAttribute(tableWikitext,attribute)
 local_,value=str_match(tableWikitext,'^{|[^\n]*'..attribute..' *= *(["\']?)([^\n]-)%1[^\n]*\n')
 ifnotvalueorvalue==''then
 value=str_match(tableWikitext,'^{|[^\n]*'..attribute..' *= *([^\n ]+)[^\n]*\n')
 end
 returnvalue
 end

 -- Get a table by id from the given wikitext
 -- @param wikitext Required. Wikitext to parse.
 -- @param id Required. Id of the table
 -- @return Wikitext of the table or nil if not found
 functionparser.getTable(wikitext,id)
 localtables=parser.getTables(wikitext)
 for_,tinipairs(tables)do
 ifid==parser.getTableAttribute(t,'id')then
 returnt
 end
 end
 end

 -- Get the data from the given table wikitext
 -- @param tableWikitext Required. Wikitext of the table to parse.
 -- @return Table data
 -- @todo Test and make more robust
 functionparser.getTableData(tableWikitext)
 localtableData={}
 tableWikitext=str_trim(tableWikitext);
 tableWikitext=str_gsub(tableWikitext,'^{|.-\n','')-- remove the header
 tableWikitext=str_gsub(tableWikitext,'\n|}$','')-- remove the footer
 tableWikitext=str_gsub(tableWikitext,'^|%+.-\n','')-- remove any caption
 tableWikitext=str_gsub(tableWikitext,'|%-.-\n','|-\n')-- remove any row attributes
 tableWikitext=str_gsub(tableWikitext,'^|%-\n','')-- remove any leading empty row
 tableWikitext=str_gsub(tableWikitext,'\n|%-$','')-- remove any trailing empty row
 forrowWikitextinstr_gsplit(tableWikitext,'|-')do
 localrowData={}
 rowWikitext=str_gsub(rowWikitext,'||','\n|')
 rowWikitext=str_gsub(rowWikitext,'!!','\n|')
 rowWikitext=str_gsub(rowWikitext,'\n!','\n|')
 rowWikitext=str_gsub(rowWikitext,'^!','\n|')
 rowWikitext=str_gsub(rowWikitext,'^\n|','')
 forcellWikitextinstr_gsplit(rowWikitext,'\n|')do
 cellWikitext=str_trim(cellWikitext)
 table_insert(rowData,cellWikitext)
 end
 table_insert(tableData,rowData)
 end
 returntableData
 end

 -- Get the internal links from the given wikitext (includes category and file links).
 -- @param wikitext Required. Wikitext to parse.
 -- @return Sequence of internal links.
 functionparser.getLinks(wikitext)
 locallinks={}
 forlinkinstr_gmatch(wikitext,'%[%b[]%]')do
 table_insert(links,link)
 end
 returnlinks
 end

 -- Get the file links from the given wikitext.
 -- @param wikitext Required. Wikitext to parse.
 -- @return Sequence of file links.
 functionparser.getFiles(wikitext)
 localfiles={}
 locallinks=parser.getLinks(wikitext)
 for_,linkinipairs(links)do
 localnamespace=str_match(link,'^%[%[ *(.-) *:')
 ifnamespaceandmw.site.namespaces[namespace]andmw.site.namespaces[namespace].canonicalName=='File'then
 table_insert(files,link)
 end
 end
 returnfiles
 end

 -- Get name of the file from the given file wikitext.
 -- @param fileWikitext Required. Wikitext of the file to parse.
 -- @return Name of the file
 functionparser.getFileName(fileWikitext)
 returnstr_match(fileWikitext,'^%[%[ *.- *: *(.-) *[]|]')
 end

 -- Get the category links from the given wikitext.
 -- @param wikitext Required. Wikitext to parse.
 -- @return Sequence of category links.
 functionparser.getCategories(wikitext)
 localcategories={}
 locallinks=parser.getLinks(wikitext)
 for_,linkinipairs(links)do
 localnamespace=str_match(link,'^%[%[ -(.-) -:')
 ifnamespaceandmw.site.namespaces[namespace]andmw.site.namespaces[namespace].canonicalName=='Category'then
 table_insert(categories,link)
 end
 end
 returncategories
 end

 -- Get the external links from the given wikitext.
 -- @param wikitext Required. Wikitext to parse.
 -- @return Sequence of external links.
 functionparser.getExternalLinks(wikitext)
 locallinks={}
 forlinkinstr_gmatch(wikitext,'%b[]')do
 ifstr_match(link,'^%[//')orstr_match(link,'^%[https?://')then
 table_insert(links,link)
 end
 end
 returnlinks
 end

 returnparser