Module:Creator

--[[  
  __  __           _       _         ____                _             
 |  \/  | ___   __| |_   _| | ___ _ / ___|_ __ ___  __ _| |_ ___  _ __ 
 | |\/| |/ _ \ / _` | | | | |/ _ (_) |   | '__/ _ \/ _` | __/ _ \| '__|
 | |  | | (_) | (_| | |_| | |  __/_| |___| | |  __/ (_| | || (_) | |   
 |_|  |_|\___/ \__,_|\__,_|_|\___(_)\____|_|  \___|\__,_|\__\___/|_|   

This module is intended to be the engine behind "Template:Creator".

Please do not modify this code without applying the changes first at 
"Module:Creator/sandbox" and testing at "Module:Creator/testcases".

Authors and maintainers:
* User:Jarekt - original version 

Handling of the fields
 ==============================================================================
 |field name        | property  | pull | push | missing  | mismatch | redundant 
 ==============================================================================
 |Name              | label     | 1    |  0   |          |          |
 |Alternative names | aliases   | 1    |  0   |          |          |
 |                  |P742,P1477,|      |      |          |          |
 |                  |P1782,P1787|      |      |          |          |
 |Sortkey           | P734,P735 | 1    |  0   |          |          | 1
 |Birthdate         | P569      | 1    |  1   |  1       | 1        | 1 
 |Deathdate         | P570      | 1    |  1   |  1       | 1        | 1 
 |Birthloc          | P19       | 1    |  1   |  1       | 1        | 1
 |Deathloc          | P20       | 1    |  1   |  1       | 1        | 1
 |Workperiod        |P2031,P2032| 1    |      |          |          |
 |                  | P1317     |      |      |          |          |
 |Workloc           | P937      | 1    |  1   |  1       | 1        | 1
 |Image             | P18       | 1    |  1   |  1       | 1        | 1
 |Homecat           | P373      | 1    |  1   |  1       | 1        | 1
 |Nationality       | P27, P172 | 1    |      |  1       | 1        | 1
 |Gender            | P21       | 1    |  1   |  1       | 1        | 1
 |Occupation        | P106      | 1    |      |  1       |          | 1
 |Linkback          | P1472     | 1    |  1   |  1       | 1        | 1
 |Wikisource        | sitelinks | 1    |  0   |          | 0        | 1
 |Wikiquote         | sitelinks | 1    |  0   |          | 0        | 1
 ===============================================================================
 
 pull - can we pull data from wikidata ? 
   - 1  - commons then wikidata
   -    - not implemented yet
   - 0  - will not implement
 push      - upload to wikidata through quick statements?
 missing   - detect if missing on Wikidata
 mismatch  - detect mismatch between wikidata and commons
 redundant - detect if redundant identical values on wikidata and commons
]]
local getLabel            = require("Module:Wikidata label")._getLabel            -- used for creation of name based on wikidata
local Wikidata_label      = require("Module:Wikidata label")                      -- used for creation of name based on wikidata
local getDate             = require("Module:Wikidata date")._date                 -- used for processing of date properties
local qualifierDate       = require("Module:Wikidata date")._qualifierDate        -- used for processing of date qualifiers
local authorityControl    = require("Module:Authority control")._authorityControl -- used for formatting of Authority control row
local alterName           = require("Module:Name")._name                          -- used for adding "option" fields to "name"
local City                = require("Module:City")                                -- used to add wikidata bases links to names of places
local ISOdate             = require("Module:ISOdate")                             -- used for internationalization of dates
local NationAndOccupation = require("Module:NationAndOccupation")._NationAndOccupation
local valid_date          = require("Module:Calendar")._valid_date
local labels              = require("Module:I18n/creator")
local TagQS               = require('Module:TagQS')
local core                = require("Module:core")
local messageBox          = require("Module:Message box")

-- ==================================================
-- === Internal functions ===========================
-- ==================================================

local function empty2nil(str)
 if str=='' then
  return nil
 else 
  return str;
 end
end

local function intersect(A, B) -- find intersection of tables A and B
 local ret = {}
 for _, a in ipairs(A or {}) do
  for _, b in ipairs(B or {}) do
   if a==b then 
    table.insert(ret, b) 
   end
  end
 end
 return ret
end

local function isodate2timestamp(dateStr)
-- convert isodate to timestamp used by quick statements
 local tStamp = nil
 if string.match(dateStr,"^%d%d%d%d$") then               -- if YYYY  format 
  tStamp = '+' .. dateStr .. '-00-00T00:00:00Z/9'
 elseif string.match(dateStr,"^%d%d%d%d%-%d%d$") then      -- if YYYY-MM format 
  tStamp = '+' .. dateStr .. '-00T00:00:00Z/10'
 elseif string.match(dateStr,"^%d%d%d%d%-%d%d%-%d%d$") then  -- if YYYY-MM-DD format 
  tStamp = '+' .. dateStr .. 'T00:00:00Z/11'
 end
 return tStamp
end

local function filepage_warningbox(text, lang, qCode)
 local boxArgs = {}
 
 boxArgs.type = 'content'
 boxArgs.text = string.format(core.langSwitch(labels[text],lang), qCode)
 
 return messageBox.main("mbox", boxArgs);
end
-- ====================================================================
-- This function is responsible for producing HTML of a single row of the template
-- At this stage all the fields are already filed. There is either one or two fields
-- INPUTS:
-- * param1 and param2 - structures for 2 fields containing fields:
--    - tag      - I18n tag used for localization of the field name. Usually name of page in MediaWiki namespace which was imported from translatewiki.org. 
--                 Alternative is to pass already translated field name.
--    - field    - field content
--    - id       - ID tag added to HTML's <td> cell. if IDs of 2 fields ar the same than we ignore the second one
--    - wrapper  - some fields need a <span class=...> wrapper around the field content 
-- ====================================================================
local function Build_html_row(param1, param2, args)
 local tag, headerCell, cell2, cell3
 local field1 = args[param1.field]
 local field2 = args[param2.field]
 if field1=='' then field1=nil; end
 if field2=='' then field2=nil; end
 if not (field1 or field2 or args.demo) then 
  return nil
 end
 if field2 then tag = param2.tag else tag = param1.tag end -- use different tag based on presence of field2
 if string.sub(tag,1,10) == 'wm-license' then
  tag = mw.message.new( tag ):inLanguage(args.lang):plain() -- label message in args.lang language
 end
 headerCell = string.format('<th scope="row">%s</th>\n', tag)
 if param1.id==param2.id then -- 2 cell row
  cell2 = string.format('<td colspan="2" class="fullwidth" id="%s">'.. param1.wrapper ..'</td>', param1.id, field1 or '')
  cell3 = ''
 else                         -- 3 cell row
  cell2 = string.format('<td class="halfwidth" id="%s">\n%s</td>', param1.id, field1 or '')
  cell3 = string.format('<td class="halfwidth" id="%s">\n%s</td>', param2.id, field2 or '')
 end
 return string.format('<tr>\n%s%s%s</tr>\n', headerCell, cell2, cell3)
end

-- ====================================================================
-- === This function is just responsible for producing HTML of the  ===
-- === template. At this stage all the fields are already filed     ===
-- ====================================================================
local function Build_html(args, cats)
 local field

 -- Top line with Creator name, lifespan and link icons -
 field = string.format('<span class="fn" id="creator"><bdi>%s\n</bdi></span>&nbsp;%s', args.name or 'missing name', args.lifespan or '') 
 if args.linkback then 
     field = string.format('%s&nbsp;[[File:Blue pencil.svg|15px|link=Creator:%s|class=skin-invert]]', field, args.linkback) 
 end
 if args.wikidata then -- Wikidata Link
  field = string.format('%s&nbsp;[[File:Wikidata-logo.svg|20px|wikidata:%s|link=wikidata:%s]]', field, args.wikidata, args.wikidata) 
 end
 if args.wikisource then --Wikisource link
  field = string.format('%s&nbsp;[[File:Wikisource-logo.svg|15px|%s|link=%s]]', field, args.wikisource, args.wikisource) 
 end
 if args.wikiquote then --Wikiquote link
  field = string.format('%s&nbsp;[[File:Wikiquote-logo.svg|15px|%s|link=%s]]', field, args.wikiquote, args.wikiquote) 
 end
 if args.QS then -- quick_statement link to upload missing info to wikidata
  field = string.format('%s&nbsp;%s', field, args.QS) 
 end

 -- Provide our own collapsible toggle in the th, which is image based
 -- This avoids depending on English for anonymous users, but it's a bit of a hack.
 local arrowtoggle = string.format(
  ' <div class="mw-collapsible-toggle %s"><span class="mw-collapsible-arrowtoggle skin-invert">&nbsp;</span></div>',
  (args.collapse or args.namespace == 6) and 'mw-collapsible-toggle-collapsed' or 'mw-collapsible-toggle-expanded'
 );
 local line = string.format('<th colspan="4">%s%s</th>', field, arrowtoggle) 
 local results = {}
 table.insert(results, string.format('<tr>\n%s\n</tr>\n', line))
 
 -- add other fields
 local param = {
  {tag='wm-license-creator-alternative-names'          , field='alternative_names', id='fileinfotpl_creator_alt-name_value'   , wrapper='<div class="nickname">\n%s</div>' },
  {tag='wm-license-creator-description'                , field='description'      , id='fileinfotpl_creator_desc_value'       , wrapper='%s' },
  {tag='wm-license-creator-date-of-birth'              , field='birthdate'        , id='fileinfotpl_creator_birthdate_value'  , wrapper='%s' },
  {tag='wm-license-creator-date-of-birth-and-death'    , field='deathdate'        , id='fileinfotpl_creator_deathdate_value'  , wrapper='%s' },
  {tag='wm-license-creator-location-of-birth'          , field='birthloc'         , id='fileinfotpl_creator_birthloc_value'   , wrapper='%s' },
  {tag='wm-license-creator-location-of-birth-and-death', field='deathloc'         , id='fileinfotpl_creator_deathloc_value'   , wrapper='%s' },
  {tag='wm-license-creator-work-period'                , field='workperiod'       , id='fileinfotpl_creator_work-period_value', wrapper='%s' },
  {tag='wm-license-creator-work-location'              , field='workloc'          , id='fileinfotpl_creator_work-location'    , wrapper='<div class="locality">\n%s</div>' },
  {tag=args.authority_tag                              , field='authority'        , id='fileinfotpl_creator_authority_value'  , wrapper='%s' },
  {tag='wm-license-artwork-references'                 , field='references'       , id='fileinfotpl_creator_references'       , wrapper='<div>\n%s</div>'}
 }
 table.insert(results, Build_html_row(param[ 1], param[ 1], args))
 table.insert(results, Build_html_row(param[ 2], param[ 2], args))
 table.insert(results, Build_html_row(param[ 3], param[ 4], args))
 table.insert(results, Build_html_row(param[ 5], param[ 6], args))
 table.insert(results, Build_html_row(param[ 7], param[ 7], args))
 table.insert(results, Build_html_row(param[ 8], param[ 8], args))
 table.insert(results, Build_html_row(param[ 9], param[ 9], args))
 table.insert(results, Build_html_row(param[10], param[10], args))
 
 -- Image on the Left
 if not args.image and args.demo then
  args.image = 'Silver - replace this image male.svg'
 end
 if args.image then --Wikiquote link
  field = string.format('[[File:%s|120x360px|alt=%s|class=photo]]', args.image, args.name or '') 
     local n = #results -- number of rows below 
  line  = string.format('<td rowspan="%i" style="width:120px" id="fileinfotpl_creator_image"><span class="wpImageAnnotatorControl wpImageAnnotatorOff">%s</span></td>', n, field) 
  table.insert(results, 2, string.format('<tr>\n%s\n</tr>\n', line) )
 end
 results = table.concat(results)
 
 -- Template styles
 -- We should make this sandbox aware
 local templatestyles = mw.getCurrentFrame():extensionTag{
  name = 'templatestyles', args = { src = 'Module:Creator/styles.css' }
 }

 -- build table
 local dir = mw.language.new( args.lang ):getDir()
 local text_align = ((dir=='ltr') and 'left') or 'right'
 local collapsed = ''
 if args.collapse or args.namespace == 6 then
  collapsed = 'mw-collapsed'
 end
 local style = string.format('class="commons-creator-table mw-collapsible %s mw-content-%s" dir="%s" lang="%s"',
  collapsed, dir, dir, args.lang)
 results = string.format('<table %s>\n%s\n</table>\n', style, results)
 results = string.format('<div class="commons-creator vcard">\n%s\n%s\n</div>\n', templatestyles, results)
 
 -- add references and documentation which are only visible in creator namespace
 if args.namespace==100 or args.demo then
  local box =''
  if args.wikidata and string.match(cats,'missing linkback') then
   box = filepage_warningbox('missing_linkback', args.lang, args.wikidata)
  elseif args.wikidata and string.match(cats,'without home category') then
   box = filepage_warningbox('missing_homecat', args.lang, args.wikidata)
  end
  local doc = mw.getCurrentFrame():expandTemplate{ title ='documentation', args = { 'Template:Creator/documentation' } }
  results = results .. box .. doc -- add documentation to pages in creator namespace
 end
 return results
end

-- ===========================================================================
-- === This function is responsible for adding maintenance categories      ===
-- === which are not related to wikidata                                   ===
-- === INPUTS:                                                             ===
-- ===  * args  - merged data from the local arguments and Wikidata        ===
-- ===========================================================================
local function add_maintenance_categories(args)
 local cats = '' -- categories 
 
 -- ====================================================
 -- === automatic tagging of pages in all namespaces === 
 -- ====================================================
 if args.type=='' or args.type=='person' then
     -- add an empty template which can be used as a tag in PetScan
  local dod  = args.deathyear or args.deathdate -- date of death
  local dob  = args.birthyear or args.birthdate -- date of birth
  local d    = os.date('!*t')                   -- current date table
  local year = tonumber(d.year)                 -- current year
  local pma  = nil                              -- years since death
  if dod then
   dod = tonumber(ISOdate._ISOyear(dod))
   if dod then
    pma = year-dod
   end
  end
  if dob and not pma then
   dob = tonumber(ISOdate._ISOyear(dob))
   if dob then
    pma = year-dob-100 -- Assumes max 100 lifespan
   end
  end

  -- Add empty tag templates to track different cases
  if pma and pma>100 then 
   mw.getCurrentFrame():expandTemplate{ title ='Works of authors who died more than 100 years ago' }
  elseif pma and pma>70 then 
   mw.getCurrentFrame():expandTemplate{ title ='Works of authors who died more than 70 years ago' }
  elseif (dod or dob or 0)>year-65 then 
   mw.getCurrentFrame():expandTemplate{ title ='Works of authors who died less than 65 years ago' }
  end
 end 
  
 -- ============================================================
 -- === automatic categorization of pages in File: namespace === 
 -- ============================================================
 if args.namespace==6 then
  if not args.image then
   mw.getCurrentFrame():expandTemplate{ title = 'Creator template without image' } -- add the template tag
  end
  return cats
 end
 
 -- ===============================================================
 -- === automatic categorization of pages in Creator: namespace === 
 -- ===============================================================
 if args.namespace~=100 then
  return cats
 end
 
 -- add [[Category:Creator templates]] category
 cats = cats .. string.format('\n[[Category:Creator templates|%s]]',args.sortkey or ' ')
 
 -- check for key information
 if not args.linkback and not args.wikidata then
  cats = cats .. '\n[[Category:Creator templates without linkback]]'
 end 
 if not args.name then
  cats = cats .. '\n[[Category:Creator templates without name]]'
 end 
 
 -- add homecat category
 if args.homecat then
  cats = cats .. string.format('\n[[Category:%s]]',args.homecat)
 end

 -- add type category
 if args.type then
  local lut = { 
   ['commons user'] = '\n[[Category:User creator templates]]',
   ['corporation']  = '\n[[Category:Corporate creator templates]]',
   ['group']        = '\n[[Category:Group creator templates]]',
  }
  cats = cats ..  (lut[args.type] or '')
  if args.type=='commons user' then
   return cats -- for commons user do not add other maintenance categories
  end
 end
 
 -- ===============================================================
 -- === automatic categorization of pages in Creator: namespace === 
 -- === all pages except: 'commons user'                        ===
 -- ===============================================================
 
 -- check for image
 if not args.image then
  cats = cats .. '\n[[Category:Creator templates without images]]'
 end
 -- check for wikidata q-code
 if not args.wikidata then
  cats = cats .. '\n[[Category:Creator templates without Wikidata link]]'
 end
 -- check for homecat
 if not args.homecat then
  cats = cats .. '\n[[Category:Creator templates without home category]]'
 else
  local hc = mw.title.new('Category:'..args.homecat)
  if not hc.exists then
   cats = cats .. '\n[[Category:Creator templates with non-existing home categories]]'
  end 
  hc = mw.title.new('Creator:'..args.homecat)
  if hc:localUrl() ~= mw.title.getCurrentTitle():localUrl() then
   cats = cats .. '\n[[Category:Creator templates with non-matching home categories]]'
  end
 end

 return cats
end


-- ===========================================================================
-- === This function is responsible for adding maintenance categories      ===
-- === to pages in category namespace                                      ===
-- === INPUTS:                                                             ===
-- ===  * args - local inputs from the creator template page               ===
-- ===========================================================================
local function add_categories_to_category_namespace(args)
 local cats
 if args.namespace~=14 or (args.homecat and mw.title.new('Category:' .. args.homecat):localUrl() ~= mw.title.getCurrentTitle():localUrl()) then
  return '' -- if not a home category than exit
 end
 local sortkey = "|" .. (args.sortkey or '')
 if #sortkey==1 then sortkey='' end
 
 cats = string.format('\n[[Category:Creator template home categories%s]]', sortkey)
 -- check for wikidata q-code
 if not args.wikidata then
  cats = cats .. '\n[[Category:Creator template home categories without Wikidata link]]'
 end

 if args.command == 'autocategorize' then
  -- add basic categories to the creator page
  cats = string.format('%s\n[[Category:People by name%s]]', cats, sortkey)
  if args.deathyear then
   cats = string.format('%s\n[[Category:%i deaths%s]]', cats, args.deathyear, sortkey)
  end
  if args.birthyear then
   cats = string.format('%s\n[[Category:%i births%s]]', cats, args.birthyear, sortkey)
  end
 end

 return cats
end

-- ===========================================================================
-- === This function is responsible for adding maintenance categories      ===
-- === to pages in creator namespace which are related to wikidata         ===
-- === INPUTS:                                                             ===
-- ===  * args0 - local inputs from the creator template page              ===
-- ===  * args1 - merge of local and wikidata metadata                     ===
-- ===  * data  - data pulled from Wikidata                                ===
-- ===========================================================================
local function add_categories_to_creator_namespace(args0, args1, data)
 local cats = ''     -- categories 
 local qsTable = {}  -- table to store QuickStatements 
 local comp    = {}  -- outcome of argument vs. wikidata comparison
 -- two forms of QuickStatements command with and without quotes
 local qsCommand = {'%s|%s|%s', '%s|%s|"%s"'}

 -- compare Linkback to the actual page name. Many "Linkbacks" are created with 
 -- tool which produces &#38; and &#39;  instead of "&" and "'"
 if args0.linkback then
  local linkback = args0.linkback
  linkback = mw.ustring.gsub(linkback, '&#39;', "'")
  linkback = mw.ustring.gsub(linkback, '&#38;', "&")
  if linkback~=args0.pagename then
   cats = cats .. '\n[[Category:Creator templates with mismatching linkback]]'
  end
 end
 
 -- add [[Category:Creator templates with unknown parameter]] category, if some parameter not on the following list is used
 local fields = {'name', 'alternative_names', 'sortkey', 'birthdate', 'deathdate', 'birthloc', 'deathloc', 'workperiod', 'workloc', 'collapse', 
         'image', 'homecat', 'nationality', 'gender', 'occupation', 'description', 'authority', 'type', 'wikisource', 'wikiquote', 'command',
   'namespace', 'linkback', 'wikidata', 'lang', 'pagename', 'reference', 'references', 'lifespan', 'birthyear', 'deathyear', 'option' }
 local set = {}
 for _, field in ipairs(fields) do set[field] = true end
 for field, _ in pairs( args0 ) do 
  if not set[field] then
   cats = string.format('%s\n[[Category:Creator templates with unknown parameter|%s]]', cats, field)
  end
 end
 
 -- add [[Category:Wikidata based creator templates]] and [[Category:Creator templates with Wikidata link: local linkback]]
 local val = {wikidata=1, option=0, linkback=0, lang=0, namespace=0, pagename=0, type=0, command=0 }
 local hash = 0;
 for field, _ in pairs( args0 ) do 
  hash = hash + (val[field] or 10)
 end
 if hash==1 then
  cats = string.format('%s\n[[Category:Creator templates based only on Wikidata|%s]]', cats, args1.sortkey or '')
 end
 
 --  if no q-code but we have "create" input argument then create new item
 if not args0.wikidata and args0.command == 'create item' then
  local description
  table.insert( qsTable, 'CREATE' )
  table.insert( qsTable, 'LAST|P31|Q5|S143|Q24731821' )  -- instance of human
  table.insert( qsTable, 'LAST|Len|"'.. args0.pagename .. '"' ) -- english label
  if args0.nationality and args0.occupation then
   local lang = args0.lang 
   args0.lang = 'en';
   description, _, _ = NationAndOccupation(args0)
   args0.lang = lang
   if args1.birthyear and args1.deathyear then
    description = string.format('%s (%s-%s)', description, args1.birthyear, args1.deathyear) 
   end
   table.insert( qsTable, 'LAST|Den|"'.. description .. '"' ) -- english description
  end
  args0.wikidata = 'LAST' 
 end
 
 -- skip the rest if no q-code
 if not args0.wikidata then
  return cats, args1
 end
 
 -- mark parameters as "local" if they are present in creator template
 local fields = {'name', 'birthdate', 'deathdate', 'birthyear', 'deathyear', 'birthloc', 'deathloc', 'image', 
  'homecat', 'nationality', 'gender', 'occupation', 'description', 'authority', 'wikisource', 'wikiquote', 'sortkey' }
 for _, field in ipairs( fields ) do
  if args0[field] then
   comp[field] = 'local'
  end
 end
 
 -- redundant if commons creator template and wikidata have those fields, without checking values
 local fields = {'wikiquote', 'wikisource' }
 for _, field in ipairs( fields ) do
  if args0[field] and data[field] then
   comp[field] = 'redundant'
  end
 end
 
 -- ==================================================
 -- === time fields  ================================= 
 -- ==================================================
 local fields = {birthdate='P569', deathdate='P570' }
 local a1, a2, d1, d2, dy
 for field, prop in pairs( fields ) do
  a1 = args0[field]        -- original creator template value often in iso (YYYY or YYYY-MM-DD) format 
  a2 = args1[field]        -- translated creator template value  
  d1 = data[field .. '_']  -- wikidata value in iso (YYYY or YYYY-MM-DD) format 
  d2 = data[field]         -- translated wikidata value
  dy = tostring(data[string.gsub(field, 'date', 'year')])  -- wikidata year value
  if a1 and not (string.match(a1,"^%d%d%d%d$") or string.match(a1,"^%d%d%d%d%-%d%d$") or string.match(a1,"^%d%d%d%d%-%d%d%-%d%d$")) then -- if YYYY or YYYY-MM-DD format 
   a1 = nil -- delete if not in iso format
  end
  
  if a1 then -- local date in iso form
   if (a1==d1) or (a2 and a2==d2) or (#a1==4 and a1==dy) then 
    comp[field] = 'redundant' -- matching iso value, translated value and commons-year matching wikidata date
   elseif d1 and a1~=d1 then 
    comp[field] = 'mismatching'
   elseif not d2 then -- missing on Wikidata 
    comp[field] = 'item missing'
   end
   -- create QS string so the Commons value can be uploded to Wikidata
   if (comp[field]=='item missing') or (#a1>4 and d1 and #d1==4 and string.sub(a1,1,4)==d1) then
    local val = isodate2timestamp(a1)
    if val then
     table.insert( qsTable, string.format(qsCommand[1], args0.wikidata, prop, val) )
    end
   end
  end
 end
 
 -- ==================================================
 -- === birthloc / deathloc place fields  ============ 
 -- ==================================================
 local fields = {birthloc='P19', deathloc='P20' }
 for field, prop in pairs( fields ) do
  local a1, a2, d1, d2, dy, _
  a2 = args0[field] -- creator template value  
  d1 = data[field]  -- wikidata q-code
  if a2 then
   a1, _ = City.qCode(a2) -- q-code for original creator template value
  end
  if d1 then
   d2 = getLabel(d1, 'en', '-') -- get english label
   dy = getLabel(d1, args0.lang) .. core.editAtWikidata(args0.wikidata, prop, args0.lang)
  end
  
  if (a1 and a1==d1) or (a2 and a2==d2)  then 
   comp[field] = 'redundant' -- matching q-code and name
  elseif (a1 and d1 and a1~=d1) or (a2 and d2 and a2~=d2) then 
   comp[field] = 'mismatching'
  elseif a1 and not d2 then -- missing on Wikidata 
   comp[field] = 'item missing'
   table.insert( qsTable, string.format(qsCommand[1], args0.wikidata, prop, a1) )
  elseif a2 and not d2 then
   comp[field] = 'item missing'
  end
  data [field..'_'] = d1 
  args0[field..'_'] = a1 
  data [field]      = dy
 end
 
 -- ==================================================
 -- === workloc field  =============================== 
 -- ==================================================
 if (args0.workloc and args0.workloc==data.workloc_en)  then 
  comp.workloc = 'redundant' -- matching q-code and name
 elseif args0.workloc and not data.workloc then -- missing on Wikidata 
  comp.workloc = 'item missing'
 end
 
 -- ==================================================
 -- === nationality and occupation  ==================
 -- ==================================================
 local fields = { nationality='nationality_', occupation='occupationEN' }
 data.nationality_ = data.nationality
 for field, field_ in pairs( fields ) do
  local a1, a2, ad
  a1 = args1[field_] -- creator template value  
  d1 = data [field_] -- wikidata q-code
  ad = intersect(a1, d1)
  
  if (a1 and d1 and #a1==#ad and (#d1==#ad or field=='occupation'))  then 
   -- for nationality all values on Commons must be the same as on Wikidata
   -- for occupation all commons values have to be on Wikidata but wikidata can have more than that
   comp[field] = 'redundant' 
  elseif (a1 and d1 and #a1>#ad) then 
   comp[field] = 'mismatching' -- some commons values are not on Wikidata
  elseif a1 and not d1 then -- missing on Wikidata 
   comp[field] = 'item missing'
  end
 end
 
 -- ==================================================
 -- === gender  =============================== 
 -- ================================================== 
 if args0.gender then -- look up q-codes of gender
  local GenderLut = { male='Q6581097', female='Q6581072'}
  a1 = GenderLut[mw.ustring.lower(args0.gender)] -- look up q-code for each gender
  d1 = GenderLut[data.gender]       -- wikidata q-code
  if a1 and d1 and a1~=d1 then 
   comp.gender = 'mismatching'
  elseif a1 and d1 and a1==d1 then 
   comp.gender = 'redundant'
  elseif a1 and not d1 then
   comp.gender = 'item missing'
   table.insert( qsTable, string.format(qsCommand[1], args0.wikidata, 'P21', a1) )  
  end 
 end
 
 -- ==================================================
 -- === odds and ends  =============================== 
 -- ================================================== 
 if args0.image then 
  args0.image_ = mw.uri.decode( args0.image, "WIKI" )
 end
 args0.linkback_ = args0.pagename;
 args0.homecat_  = args0.homecat; 
 
 local fields = {image='P18', linkback='P1472', homecat='P373'}
 for field, prop in pairs( fields ) do
  a1 = args0[field..'_'] -- creator template value  
  d1 = data[field]       -- wikidata q-code
  if a1 and d1 and a1~=d1 then 
   comp[field] = 'mismatching'
  elseif a1 and d1 and a1==d1 then 
   comp[field] = 'redundant' 
  elseif a1 and not d1 then
   comp[field] = 'item missing'
   table.insert( qsTable, string.format(qsCommand[2], args0.wikidata, prop, a1) )
  end 
 end
 if comp.linkback == 'redundant' and (hash~=1 or not args0.linkback) then
  comp.linkback = nil
 end
 if args0.sortkey and data.sortkey and args0.sortkey==data.sortkey then
  comp.sortkey = 'redundant'
 end
 if args0.description and args1.description_==args0.description then -- description is "French painter" while nationality is FR and occupation is "painter"
  comp.description = 'redundant'
 end
 
 -- ==================================================
 -- === alter look of some fields === 
 -- ==================================================
 local fields = {'birthloc', 'deathloc', 'birthdate', 'deathdate' }
 for _, field in ipairs( fields ) do
  if ( comp[field] == 'mismatching' ) or ( comp[field] == 'local' and data[field] ) then
   args1[field] = string.format('<div style=\"background-color:PeachPuff\">%s</div> <br/>%s', args1[field], data[field])
  elseif ( comp[field] == 'redundant' ) then
   args1[field] = string.format('<div style=\"background-color:Thistle\">%s</div>', args1[field])
  elseif ( comp[field] == 'item missing' and args1[field])  then
   args1[field] = string.format('<div style=\"background-color:PeachPuff\">%s</div>', args1[field])
  end
 end

 -- ==================================================
 -- === Create categories and QuickStatement codes === 
 -- ==================================================
 
 -- create categories based on comp structure
 for field, outcome in pairs( comp ) do
  cats = string.format('%s\n[[Category:Creator templates with Wikidata link: %s %s]]', cats, outcome, field)
 end
 
 -- convert QS table to a string
 local QS   = ''     -- quick_statements final string
 if #qsTable>0 then
  local today = '+' .. os.date('!%F') .. 'T00:00:00Z/11' -- today's date in QS format
     local url   = mw.title.getCurrentTitle():canonicalUrl()
     local source = '|S143|Q24731821|S813|' .. today .. '|S4656|"' .. url .. '"'
  local qsWrapper = '&nbsp;[[File:Commons_to_Wikidata_QuickStatements.svg|15px|link=%s]]'
  QS = table.concat( qsTable, source..'||') .. source    -- combine multiple statements into a single command separated by ||
  QS = mw.ustring.gsub(QS, ' ', "%%20")
  QS = mw.ustring.gsub (mw.uri.encode(QS),'%%2520','%%20')
  QS = 'https://quickstatements.toolforge.org/#/v1=' .. QS    -- create full URL link
  QS = string.format(qsWrapper, QS)
  cats = cats .. '\n[[Category:Creator templates with Wikidata link: quick statements]]'
 end
 args1.QS = QS
 
 -- tag invalid dates that use YYYY-MM-DD format
 if (args0.deathdate and (not valid_date(args0.deathdate))) or 
    (args0.birthdate and (not valid_date(args0.birthdate))) then
  cats = cats .. '\n[[Category:Pages with incorrect date]]'
 end
 
 return cats, args1
end


-- ===========================================================================
-- === Harvest wikidata properties matching creator template fields ==========
-- ===========================================================================

local function getPropertyQual(entity, prop, qualifiers, lang, offset)
 local Res = {}
 if entity.claims and entity.claims[prop] then
  for k, statement in ipairs( entity:getBestStatements( prop )) do
   if (statement.mainsnak.snaktype == "value") then 
    local res = {} -- table with fields: key, value, P... (qualifiers)
    local jdn = k -- "Julian day number" will be used as a key for sorting events; initialize
    local val = statement.mainsnak.datavalue.value.id
    val = getLabel(val, lang)
    res.value = val
    for iQual, qual in ipairs( qualifiers ) do
     if statement.qualifiers and statement.qualifiers[qual] then
      local snak = statement.qualifiers[qual][1]
      if (snak.snaktype == "value" and snak.datatype == 'time') then
       val = qualifierDate(snak, lang)
       if iQual==1 then -- first qualifier in the qualifiers list will be used as a sorting value
        jdn = val.jdn
       end
       val = val.str
      end
      res[qual] = val
     end
    end
    res.key = jdn
    table.insert(Res, res)
   end
  end
 end
 local tableComp = function (rec1, rec2) return rec1.key<rec2.key end
 table.sort(Res, tableComp)
 return Res
end

-- ===========================================================================
local function get_work_location(entity, lang)
 -- work_location (P937) /  'P580', 'P582' (time properties) 
 local prop = getPropertyQual(entity, 'P937', {'P580', 'P582', 'P585'}, lang)
 local X={}
 for _, p in ipairs(prop) do
  local str = p.value
  if p.P580 or p.P582 then
   str = string.format("%s (%s&ndash;%s)", p.value, p.P580 or '', p.P582 or '')
  elseif p.P585 then
   str = string.format("%s (%s)", p.value, p.P585)
  else
   str = p.value
  end
  table.insert(X, str)
 end
 if #X>0 then
  return table.concat(X,"; ")
 end
 return nil
end

-- ===========================================================================
local function editAtWikidata(data, qCode, property,  lang)
 for prop, field in pairs( property ) do
   if data[field] then
   data[field] = data[field].. core.editAtWikidata(qCode, prop,  lang)
   end
 end
 return data
end

-- ===========================================================================
local function harvest_wikidata(qCode, lang, namespace, pagename)
 local str, d
 local data = {} -- structure similar to "args" but filled with wikidata data
 local cats = ''
 local entity = nil
 if mw.wikibase and qCode then
  entity = mw.wikibase.getEntity(qCode)
  if not entity then
    cats = '[[Category:Creator templates with bad Wikidata link|invalid]]' 
   elseif entity.id~=qCode then
    cats = '[[Category:Creator templates with redirected Wikidata link]]' 
    qCode = entity.id
  end
 end
 if not entity then
  return data, cats
 end

 -- ===========================================================================
 -- === Step 1: time properties
 -- =========================================================================== 
 -- harvest time properties: translated date and year number
 local d1 = getDate(entity, 'P569' , lang)
 local d2 = getDate(entity, 'P570' , lang)
 local d3 = getDate(entity, 'P1636', lang) 
 local d4 = getDate(entity, 'P4602', lang) 
 data.birthdate, data.birthdate_, data.birthyear = d1.str, d1.iso, d1.year
 data.deathdate, data.deathdate_, data.deathyear = d2.str, d2.iso, d2.year
 data.baptism,   data.baptismyear                = d3.str,         d3.year
 data.burial,    data.burialyear                 = d4.str,         d4.year
 data = editAtWikidata(data, qCode, { P569='birthdate', P570='deathdate'},  lang)
 
 -- baptism date as birth date
 if not data.birthdate and data.baptism then 
  data.birthdate = mw.getCurrentFrame():expandTemplate{ title='Lifetime date', args={'baptism', data.baptism, lang=lang} } .. core.editAtWikidata(qCode, 'P1636', lang)
  data.birthyear = data.baptismyear
 end
 -- burial date as death date
 if not data.birthdate and data.baptism then 
  data.deathdate = mw.getCurrentFrame():expandTemplate{ title='Lifetime date', args={'buried', data.burial, lang=lang} } .. core.editAtWikidata(qCode, 'P4602', lang)
  data.deathyear = data.burialyear
 end
 data.birthyear = tostring(data.birthyear or '')
 data.deathyear = tostring(data.deathyear or '')

 -- workperiod
 local property = { P2031='workperiod1', P2032='workperiod2', P1317='workperiod'}
 for prop, field in pairs( property ) do
   d1 = getDate(entity, prop, lang)
   if d1.str then
   data[field..'_'] = d1.str
   data[field] = d1.str .. core.editAtWikidata(qCode, prop,  lang)
   end
 end
 if not data.workperiod and (data.workperiod1 or data.workperiod2) then 
  data.workperiod = (data.workperiod1  or '') .. '&ndash;' .. (data.workperiod2  or '')
  data.workperiod_= (data.workperiod1_ or '') .. '&ndash;' .. (data.workperiod2_ or '')
 end
 data.workloc    = get_work_location(entity, lang) 
 if data.workloc then
  data.workloc = data.workloc .. core.editAtWikidata(qCode, 'P937',  lang)
 end
 data.workloc_en = get_work_location(entity, 'en')
 
 -- lifespan displayed after name
 if data.birthyear~='' or data.deathyear~='' then 
  data.lifespan = string.format('(%s&ndash;%s)', data.birthyear, data.deathyear)
 elseif data.workperiod_ then -- create from work period (without "edit at wikidata")
  data.lifespan = string.format('([[d:Q36424|fl.]] %s)', data.workperiod_)  
 end
  
 -- ===========================================================================
 -- === Step 2: simple string and Q-code properties
 -- =========================================================================== 
 -- harvest string and Q-code properties
 local property = {P18='image', P19='birthloc', P20='deathloc',
  P109='signature', P373='homecat', P734='lastname', P735='firstname',
  P742='pseudonym', P1472='linkback', P1477='birthname',
  P1782='courtesyname', P1787='artname'}
 for prop, field in pairs( property ) do
  if entity.claims and entity.claims[prop] then -- if we have wikidata item and item has the property
   -- capture single "best" Wikidata value
   for _, statement in pairs( entity:getBestStatements( prop )) do
    if (statement.mainsnak.snaktype == 'value') then
     local v = statement.mainsnak.datavalue.value
     if v.id then v = v.id end
     data[field] = v
    end
   end
  end
 end
 
 -- if homecat missing than look for it among sitelinks
 if not data.homecat then
  local sitelink
  if entity.sitelinks and entity.sitelinks.commonswiki then
   sitelink = entity.sitelinks.commonswiki.title
  end
  if not sitelink then -- check for "topic's main category" sitelinks
   local cat_id = (core.parseStatements(entity:getBestStatements( "P910" ), nil) or {nil})[1]
   if cat_id then
    sitelink = Wikidata_label._sitelinks(cat_id, "commons")['']
   end
  end  
  if sitelink and mw.ustring.sub(sitelink,1,9)=='Category:' then
   data.homecat = mw.ustring.sub(sitelink,10)
  end
 end
 
 -- get "sortkey" field
 if not data.sortkey then
  local lastname, firstname, name_part
  if data.lastname then
   lastname = getLabel(data.lastname, lang, '-')
  elseif namespace == 100 then
   name_part = mw.text.split(pagename, '%(')
   name_part = mw.text.trim (name_part[1])
   name_part = mw.text.split(name_part, ' ')
   lastname  = name_part[#name_part]
  else
   lastname  = "ZZZ"
  end
  data.lastname = lastname
  if data.firstname then
   firstname = getLabel(data.firstname, lang, '-')
  else
   firstname = data.linkback or ''
  end
  data.sortkey =  lastname .. ', ' .. firstname 
 end
 
 -- convert gender
 data.gender_ = data.gender
 if data.gender=='Q6581097' or data.gender=='Q2449503' then
  data.gender = 'male'
 end
 if data.gender=='Q6581072' or data.gender=='Q1052281' then
  data.gender = 'female'
 end
 data.image = data.image or data.signature
 
 -- =================================================================================
 -- === Step 5: name, wikisource, wikiquote, alternative_names and authority control
 -- ================================================================================= 
 -- get name field
 data.name = getLabel(entity, lang, 'wikipedia') -- create name based on wikidata label

 -- prepare fallback list of languages
 local langList = mw.language.getFallbacksFor(lang)
 table.insert(langList, 1, lang)

 -- get alternative names
 local altNameTable = {}
 local nameTypes = {birthname='birth', pseudonym='pseudonym',
  courtesyname='courtesy name', artname='artist'}
 for t,operation in pairs(nameTypes) do
  if data[t] then
   if data[t].text then
    data[t] = data[t].text
   end
   local nameStr = alterName(operation, data[t], lang)
   table.insert(altNameTable, nameStr)
  end
 end
 for _, lng in ipairs(langList) do 
  local aliasTable = Wikidata_label._aliases(entity, lng)
  if #aliasTable>0 and #aliasTable<8 then -- skip aliases if more than 8 of them
   for _,alias in pairs(aliasTable) do
    local isSpecialName = false
    for t,_ in pairs(nameTypes) do
     if alias == data[t] then
      isSpecialName = true
     end
    end
    if not isSpecialName then
     table.insert(altNameTable, alias)
    end
   end
   data.alternative_names = table.concat(altNameTable, '; ')
   break 
  end
 end
 
 -- get wikisource and wikiquote link
 local projects = {s='wikisource', q='wikiquote'}
 for code, project in pairs(projects) do
  local sitelinks = Wikidata_label._sitelinks(entity, project)
  if sitelinks then
   local lng, _ = next(sitelinks)    -- get language of the first sitelink
   table.insert(langList, lng) -- and add it to the list so there is at least one lang with sitelink on the list
   for _, language in ipairs(langList) do 
    local sitelink = sitelinks[language]
    if sitelink then 
     data[project] = string.format('%s:%s:%s', code, language, sitelink)
     break 
    end
   end
  end
 end   
 
 -- get authority control template  
 local AC_cats
 local nIdent = nil -- number of authority control identifiers to display (nil means unlimited)
 if namespace == 6 then 
  nIdent = 5     -- limit number of identifiers in file namespace for clarity
 end
 data.authority, AC_cats = authorityControl(entity, {wikidata = qCode}, lang, nIdent) 
 if not (namespace == 2 or namespace == 6 or namespace == 828 or math.fmod(namespace,2)==1) then
  cats = cats .. AC_cats -- lets not add authorityControl categories to user pages, files, modules or talk pages and concentrate on templates and categories instead
 end
 
 return data, cats
end

-- ==================================================
-- === External functions ===========================
-- ==================================================
local p = {}

-- ===========================================================================
-- === Version of the function to be called from other LUA codes
-- ===========================================================================
function p._creator(args)
 local lang = args.lang  -- user's language
 local cats = ''         -- categories 
 local str, data
 
 -- look up title info
 local title    = mw.title.getCurrentTitle()
 args.namespace = title.namespace   -- get page namespace
 args.pagename  = title.text        -- get {{PAGENAME}}
 
 -- ===========================================================================
 -- === Step 1: clean up of template arguments "args"
 -- ===========================================================================
 args.type = string.lower(args.type or 'person')   -- if 'type' field is not specified than set to "person"
 if args.linkback then
  args.linkback = string.sub(args.linkback,9)
 end

 -- clean up "gender" field
 if string.sub(args.gender or '',1,1)=='m' then args.gender=  'male' end
 if string.sub(args.gender or '',1,1)=='f' then args.gender='female' end
 
 --make a copy of args structure to capture raw inputs
 local args0 = {} -- original args
 for name, value in pairs( args ) do 
  args0[name] = value
 end 
 
 --get birthyear and deathyear from full dates
 if args.birthdate then
  args.birthyear = empty2nil(ISOdate._ISOyear(args.birthdate))
  args.birthdate = ISOdate._ISOdate(args.birthdate, lang)
 end
 if args.deathdate then
  args.deathyear = empty2nil(ISOdate._ISOyear(args.deathdate))
  args.deathdate = ISOdate._ISOdate(args.deathdate, lang)
 end
 
 -- ===========================================================================
 -- === Step 2: one by one merge wikidata and creator data
 -- ===========================================================================
 data, cats = harvest_wikidata(args.wikidata, lang, args.namespace, args.pagename)
 local description, args1, data1 = NationAndOccupation(args)
 local fields = {'nationality', 'occupation', 'gender', 'occupationEN'}
 for _, field in ipairs( fields ) do 
  args[field] = args1[field] 
  data[field] = data1[field]
 end
 args.nationality_ = args.nationality
 
 -- mass merge (prioritize local values)
 fields = {'name', 'alternative_names', 'sortkey', 'birthdate', 'deathdate', 'birthloc', 'deathloc', 'workperiod',
         'image', 'homecat', 'nationality', 'gender', 'occupation', 'authority',  'wikisource', 'wikiquote', 'workloc', 
     'linkback', 'lifespan', 'birthyear', 'deathyear', 'collapse' }
 for _, field in ipairs( fields ) do 
  args[field] = args[field] or data[field]
 end
 
 -- process "name" field
 if args.option and args.option~='' then -- modify name based on "option" parameter
  local base_name = args.name
  -- call [[module:Name]] with the task
  args.name = alterName(args.option, args.name, lang)
  if args.name  == "name not supported" then 
   args.name = base_name
   cats = cats .. '\n[[Category:Bad use of creator template - option]]'
  end
 end
  
 -- process places fields
 -- locations can be words or q -codes. Add links 
 args.birthloc = City._city(args.birthloc, lang) 
 args.deathloc = City._city(args.deathloc, lang)
 if args.workloc and not string.find(args.workloc, ' ') then
  args.workloc = City._city(args.workloc, lang) -- single word workloc will get a link
 end
 
 -- lifespan displayed after name
 if args.lifespan then
  args.lifespan = string.gsub(args.lifespan, '-', '&ndash;') -- use special dash
 end

 -- process "Authority Control" field
 args.authority_tag = getLabel("Q36524", args.lang, "wikipedia", "ucfirst")
 
 -- process "description" field
 -- Add phrase like "French painter" to the description field
 description = mw.text.trim(description)
 if description and #description>0 then
  if args.description then
   args.description_=  description
   args.description =  description .. '<br/>' .. args.description
  else
   args.description =  description
  end
 end
  
 -- use Normalization Form D to convert string with accented characters to more sort friendly format
 -- See http://unicode.org/reports/tr15/ for examples
 args.sortkey  = mw.ustring.toNFD(args.sortkey or '')
 
 -- references are only shown in ''Creator'' namespace
 if args.namespace~=100 then 
  args.references = nil
 end
 
 -- convert all empty strings to nils
 for _, field in ipairs( fields ) do 
  if args[field] == '' then 
   args[field] = nil; 
  end
 end
 
 -- ===========================================================================
 -- === Step 3: create maintenance categories and render html of the table
 -- ===========================================================================
 if args.namespace==14 and (args.type=='' or args.type=='person') then
  cats = cats .. add_categories_to_category_namespace(args)
 end
 cats = cats .. add_maintenance_categories(args)
 
 -- If creator namespace and "person" template than add maintenance categories
 args.QS = nil;
 if args.namespace==100 and (args.type=='' or args.type=='person') then
  str, args = add_categories_to_creator_namespace(args0, args, data)
  cats = cats .. str
 end
 local results = Build_html(args, cats)
 return results, cats
end

-- ===========================================================================
-- === Version of the function to be called from template namespace
-- ===========================================================================
function p.creator(frame)
 -- switch to lowercase parameters to make them case independent
 local args = core.getArgs(frame)

 -- alias field names
 args.references = args.references or args.reference      -- two alternative names for references
 
 -- parse args.option field, which is passed through individual Creator template (page in Creator namespace)
 local options = mw.text.split(args.option or '', '/') -- individual keywords can be separated by "/"
 args.option = nil
 for _, option in pairs( options ) do
  if option == 'autocategorize' then
   args.command = option -- some "options" are to modify the name and some are commands to do things
  elseif option == 'collapse' then
   args.collapse = 1 -- some "options" are to modify the name and some are commands to do things
  elseif #option>3 then
   args.option = option
  end
 end
 if args.wikidata == "create" then
  args.command  = "create item"
  args.wikidata = nil
 end
 
 -- Create invisible language independent marking in format similar to QuickStatements code based on Wikidata and Option
 local QS = ''
 if args.wikidata and string.match(args.wikidata or '', "^Q%d+$") then -- invisible language independent marking
  if not args.option then -- no "option" modifier
   QS = TagQS.createTag('creator', 'P170', args.wikidata)
  else
   args.option = mw.ustring.gsub(mw.ustring.lower(args.option), '_', ' ') -- normalize input string
      -- Not handled options: "studio of", "or follower", "or workshop", "and workshop",  "formerly attributed to"
   local qual1 = {['workshop of']='P1774', ['follower of']='P1775', ['circle of']='P1776', 
            ['manner of']='P1777', ['school of']='P1780', ['after']='P1877'} -- ['possibly']='P1779', 
   local qual2 = {['probably']='Q56644435', ['presumably']='Q18122778', ['possibly']='Q30230067', ['attributed to']='Q230768'}
   if qual1[args.option] then -- use anonymous = Q4233718 with qualifier
    QS = TagQS.createTag('creator', 'P170', string.format('Q4233718,%s,%s', qual1[args.option], args.wikidata))
   elseif qual2[args.option] then -- use "nature of statement" (P5102) qualifier
    QS = TagQS.createTag('creator', 'P170', string.format('%s,P5102,%s', args.wikidata, qual2[args.option]))
   end
  end
 end
 
 -- call the inner "core" function
 local results, cats = p._creator(args) 
 return results .. QS .. cats
end

return p