Wikipedia:ReFill/technical

Add links
From Wikipedia, the free encyclopedia

This page documents the architecture of reFill.

Components[edit]

Toolbar gadget[edit]

Reflinks.js is a gadget that provides a way to submit a page to reFill in a single click.

Front end[edit]

This is a Vue.js single-page application where the user can enter the wiki page to process, select options, submit the page to the reFill API and check the results.

The front end code is bundled using webpack.

Middle layer[edit]

Celery is use to queue requests. Key-value data is held in memory using the Redis data store.

Back end[edit]

The back end code is written in Python and uses the Flask web framework. Docker provides containerization. Citoid is the main parser, called from citoid.py. Citoid is maintained by the Wikimedia Foundation and uses Zotero. The relationship between Citoid and Zotero is described in a tech talk.

Rather than reFill containing code to cope with the vagaries of thousands of different websites, Zotero translators can be created, benefiting other referencing tools used on Wikipedia.

Hosting[edit]

The production Docker container is hosted on the Toolforge PaaS environment, which uses Kubernetes.

contents

/mnt/nfs/labstore-tools-project/refill 

config
  /banners.php
  /citoid.php
  /common.php
  /README.md
  /stable.php
  /test.php
Git
  /date
    /src
      /Lang
      /Date.php
      /DateServiceProvider.php
    /tests
      /AutomaticTest.php
      /DateTest.php
      /TranslationKaTest.php
      /TranslationTest.php
    /composer.json
    /generator.php
    /LICENSE
    /phpunit.xml
    /README.md
logs [EMPTY]
old-ng [EMPTY]
refill-ng-dist [EMPTY]
versions
    /citoid
      /backend
      /bower_components
        /bootstrap
        /chosen
        /jquery
      /config
        /config.php
      /docker
        /refill-api
          /Dockerfile
        /refill-worker
          /Dockerfile
      /messages
      /src
        /Intuition
      /vendor
      /web
      /build-images.sh
      /LICENSE.md
      /Makefile
      /README.md
      /start.sh
    /stable
      /config
      /messages
      /src
      /templates
      /tests
        /Reflinks
      /vendor
      /web
        /components
        /hooks
        /ng
        /ng.old
        /scripts
        /style
        /acknowledgments.php
        /api.php
        /index.html
        /phpinfo.php
        /result.php
        /test.php
      /bower.json
      /composer.json
      /LICENSE
      /phpunit.xml.dist
      /Procfile
      /README.md
      /toolinfo.json
    /test
dist.tar.gz
index.html
README
refill2.tgz
replica.my.cnf
service.log
service.manifest
service.template
stats.php
test.php
uniqua.json
contents

/mnt/nfs/labstore-tools-project/refill-api 

 /archived
    /refill.yml
    /probe.sh
    /run-probe.sh
    /refill.yaml.old
    /refill-new.yaml
    /deployment.yml
 /refill
    /backend
       /apicache-py3
       /__pycache__
          /celeryconfig.cpython-37.pyc
          /celeryconfig.cpython-34.pyc
       /refill
          /utils
          /transforms
          /dataparsers
          /__pycache__
          /models
          /formatters
          /tasks.py
          /throttle.ctrl
          /__init__.py
       /throttle.ctrl
       /celerybeat-schedule.db
       /celeryconfig.py
       /requirements.txt
       /Pipfile.lock
       /Pipfile
       /app.py
       /pywikibot.lwp
       /celeryconfig.example.py
       /celeryconfig.docker.py
    /web
       /src
       /libs
       /webpack.config.js
       /package-lock.json
       /package.json
       /config.development.js
       /yarn.lock
       /template.html
       /README.md [contains build instructions for prod]
    /messages
    /docker
       /refill-worker
          /Dockerfile
       /refill-api
          /Dockerfile
    /refill
    /apicache
    /__pycache__
       /celeryconfig.cpython-34.pyc
    /worker-logs.txt
    /README.md
    /Makefile
    /start.sh
    /build-images.sh
    /nohup.out
    /throttle.ctrl
    /LICENSE.md
 /www
    /python
       /venv
          /bin
          /share
          /lib
             /python3.7
                /site-packages
          /include
          /pyvenv.cfg
 /logs [empty]
 /uwsgi.log
 /worker-deployment.yml
 /scheduler.sh
 /service.manifest
 /worker.sh
 /service.template
 /requirements.txt (10/1/2019)
 /replica.my.cnf
contents

/mnt/nfs/labstore-tools-project/refill-api-test 

 /logs [EMPTY]
 /refill
   /backend
     /__pycache__
     /apicache-py3
     /refill
       /__pycache__
       /dataparsers
          /__pycache__
          /__init__.py
          /archiveis.py
          /arxiv.py
          /badauthors.py
          /baddates.py
          /citoid.py
          /detectdeadlink.py
          /doi.py
          /download.py
          /jstor.py
          /ncbi.py
          /parsespecial.py
          /parseurl.py
          /README.md
          /ris.py
          /title.py
       /formatters
          /__pycache__
          /__init__.py
          /citetemplate.py
          /formatter.py
          /plainlink.py
       /models
          /__pycache__
          /__init__.py
          /citation.py
          /context.py
       /transforms
          /__pycache__
          /__init__.py
          /fillexternal.py
          /fillref.py
          /mergeref.py
          /transform.py
       /utils
          /__pycache__
          /__init__.py
          /errors.py
          /parser.py
          /test_parser.py
          /utils.py
       /__init__.py
       /tasks.py
       /throttle.ctrl
     /app.py
     /celeryconfig.docker.py
     /celeryconfig.example.py
     /celeryconfig.py
     /Pipfile
     /Pipfile.lock
     /pywikibot.lwp
     /requirements.txt
     /throttle.ctl
   /docker
       /refill-worker
          /Dockerfile
       /refill-api
          /Dockerfile
   /messages
   /web
     /libs
        /wdiff.js
     /src
        /components
          /Change.vue
          /ChangeDetails.vue
          /ErrorDetails.vue
          /PreferencesEditor.vue
        /pages
          /Index.vue
          /LegacyBridge.vue
          /PageNotFound.vue
          /Result.vue
        /App.vue
        /main.js
        /Store.js
        /Utils.js
     /config.development.js
     /package.json
     /package-lock.json
     /README.md
     /template.html
     /webpack.config.js
     /yarn.lock
   /build-images.sh
   /LICENSE.md
   /Makefile
   /README.md
   /start.sh
 /www
   /python
      /venv
 /replica.my.cnf
 /service.manifest
 /uwsgi.log
 /worker.sh
 /worker-deployment.xml

Access to the hosting environment is via PuTTY and WinSCP - PuTTY for SSH command line access, and WinSCP for file transfer.

How to's[edit]

Discussions and links[edit]

Retrieved from "https://en.wikipedia.org/w/index.php?title=Wikipedia:ReFill/technical&oldid=1186337670"