config_unix.yaml

---
# application main params
general:
    # where all paths and following files are located
    basedirectory: .
    locale: en_US.UTF-8
    # following directories or files a under "basedirectory"
    index: index
    dbenv: db
    # log options, file is "pytextminer-log.txt"
    logsize: 1024000
    loglevel: debug
    user: sessions
    shared: shared
    whitelist_directory: whitelists
    source_file_directory: source_files
    userstopwords: user_stopwords.csv
    userwhitelist: user_whitelist.csv
    # stopwords are under share
    stopwords: stopwords/en.txt
# extraction settings
datasets:
    doc_extraction:
        # the following values can be document object fields:
        # - defined by one of the field's value
        # - constants required fields : 'content'/'label'/'id'
        - title
        - content
        #- keywords
    # tina csv columns declaration
    # will ignore undeclared fields
    # and warn not found optional fields
    tinacsv:
        # doc_label represents one of the field's key
        # if not found in the file, will use the field specified by "label"
        doc_label: acronym
        fields:
            # required fields
            label: doc_id
            content: abstract
            corpus_id: corp_id
            id: doc_id
            # optionnal fields
            title: title
            acronym: acronym
            keywords: keywords
        # csv reader params
        encoding: utf_8
        #
        # CSV dialects. 
        #
        # Possible choices: 
        #   auto, excel, excel-tab, excel-comma, excel-semicolon
        #
        # auto: will try to detect when importing a csv file
        # excel: Microsoft Excel will use current LOCALE to determine either "," or ";"
        # excel-tab: tab-separated
        # excel-comma: comma-separated
        # excel-semicolon: semicolon-separated
        #
        # Guidelines:
        #   
        # dialect_read: used for importing a CSV file. Maybe you want to leave this to "auto"
        #               except if your spreadsheet messed something, and indexing fails
        #
        # dialect_write: used to write to a CSV file. If "auto" don't work, you HAVE to edit this.

        dialect_read: auto
        dialect_write: auto
    # pubmed.gov "medline" file export
    medline:
        # doc_label represents one of the field's key
        # if not found in the file, will use the field specified by "label"
        doc_label: title
        fields:
            label: PMID
            content: AB
            corpus_id: DP
            id: PMID
            # optional fields
            title: TI
        # period_size = number first characters of pub date field "corpusField"
        period_size: 4
        encoding: ascii
    # archive of pubmed.gov "medline" files, organized like this
    # Medline/
    #    period1/period1.txt
    #    period2/period2.txt
    #medlinearchive:
        # TODO
    # extraction size
    ngramMin: 1
    ngramMax: 4
    # tagger learning on 2 x training_tagger_size sentences of tagged nltk corpus
    # to train on the whole corpora, mark ~ instead of a number of phrases
    training_tagger_size: ~
    # under "basedirectory" directory, delete this file to regenerate a new one on next starting
    tagger: shared/tagger.pickle
    # change this Reg Expression to change NGram extraction filtering
    postag_valid: '^((VBD,|VBG,|VBN,|CD.?,|JJ.?,|\?,){0,2}?(N.?.?,|\?,)+?(CD.,)??)+?((PREP.?|DET.?,|IN.?,|CC.?,|\?,)((VBD,|VBG,|VBN,|CD.?,|JJ.?,|\?,){0,2}?(N.?.?,|\?,)+?)+?)*?$'
datamining:
    template: shared/gexf/gexf.default.template
    # default values if no params are passed to gexf exporter
    DocumentGraph:
        edgethreshold:
            - 0.000
            - inf
        nodethreshold:
            - 0.00
            - inf
        #proximity: sharedNGrams
        proximity: logJaccard
        maxdegree: 100
    NGramGraph:
        edgethreshold:
            - 0.0000
            - inf
        nodethreshold:
            - 0.00
            - inf
        alpha: 0.10
        #hapax: 1
        proximity: Cooccurrences
        #proximity: EquivalenceIndex
        #proximity: PseudoInclusion
indexer:
    minCooc: 10