attachment:genstat.py of genstat.py

   1 # -*- coding: utf-8 -*-
   2 """
   3     pygments.lexer.genstat
   4     ~~~~~~~~~~~~~~~~~~~~~
   5 
   6     Lexer for GenStat Release 10.
   7 
   8     Note: in July 2010 Genstat 13 was released, although new procedure and
   9     functions have been added, syntax remains the same. This lexer should
  10     still work and only minor changes needed to update it.
  11 
  12     :copyright: Copyright 2011 by Ian Riley, ian.riley@adelaide.edu.au
  13     :copyright: Copyright 2006-2010 by the Pygments team, see AUTHORS.
  14     :license: BSD, see LICENSE for details.
  15     
  16 """
  17 
  18 import re
  19 
  20 from pygments.lexer import RegexLexer, bygroups, include
  21 from pygments.token import Punctuation, Text, Comment, Operator, Keyword,\
  22                            Name, String, Number
  23 
  24 __all__ = ['GenStatLexer']
  25 
  26 class GenStatLexer(RegexLexer):
  27     """
  28     
  29     Pygments lexer for GenStat Release 10 
  30 
  31     GenStat is a registered trade mark in certain jurisdictions of the Lawes
  32     Agricultural Trust, Rothamsted. All rights reserved.
  33 
  34     VSN International, 5 The Waterhouse, Waterhouse Street, Hemel Hempstead,
  35     Hertfordshire HP1 1ES, UK
  36 
  37     info@genstat.co.uk 
  38     http://www.genstat.co.uk/
  39 
  40     """
  41 
  42     name = 'Genstat'
  43     aliases = ['genstat','gen']
  44     filenames = ['*.gen']
  45     mimetypes = ['text/x-genstat']
  46 
  47     flags = re.MULTILINE | re.IGNORECASE
  48 
  49     normalise = False # if True all commands will display in caps and in full
  50     # if abbreviated words are normalise, this may affect line width and
  51     # wrapping
  52 
  53     continued = False
  54 
  55     declarations = 'DECLARE|DIAGONALMATRIX|DUPLICATE|DUMMY|EXPRESSION|FACTOR|\
  56 FORMULA|LRV|MATRIX|PDUPLICATE|POINTER|SCALAR|SSPM|STRUCTURE|\SYMMETRICMATRIX|\
  57 TABLE|TEXT|TREE|TSM|VARIATE'
  58 
  59     flowcontrols = 'BREAK|CALLS|CASE|DEBUG|ELSE|ENDBREAK|ENDCASE|ENDDEBUG|\
  60 ENDIF|ENDJOB|ENDFOR|ENDPROCEDURE|EXECUTE|EXIT|FOR|IF|JOB|OPTION|OWN|PARAMETER|\
  61 PASS|PROCEDURE|STOP|SUSPEND|WORKSPACE'
  62 
  63     directives = 'ADD|ADDPOINTS|ADISPLAY|AFRESPONSESURFACE|AKEEP|ANOVA|ASSIGN|\
  64 AXES|BARCHART|BASSESS|BCUT|BJOIN|BGROW|BLOCKSTRUCTURE|BREAK|CALCULATE|\CALLS|\
  65 CAPTION|CASE|CATALOGUE|CLOSE|CLUSTER|COKRIGE|COLOUR|COMBINE|CONCATENATE|\
  66 CONTOUR|COPY|CORRELATE|COVARIATE|CVA|DCLEAR|DCONTOUR|DDISPLAY|DEBUG|\
  67 NSTATEMENTS|DECLARE|DELETE|DEVICE|DFINISH|DGRAPH|DHISTOGRAM|DIAGONALMATRIX|\
  68 DISPLAY|DISTRIBUTION|DKEEP|DLOAD|DPIE|DREAD|DROP|DSAVE|DSHADE|DSTART|DSURFACE|\
  69 DUMMY|DUMP|DUPLICATE|D3GRAPH|D3HISTOGRAM|EDIT|ELSE|ELSIF|ENDBREAK|ENDCASE|\
  70 ENDDEBUG|ENDFOR|ENDIF|ENDJOB|ENDPROCEDURE|ENQUIRE|EQUATE|ESTIMATE|EXECUTE|\
  71 EXIT|EXPRESSION|FACROTATE|FACTOR|FAULT|FCLASSIFICATION|FCOVARIOGRAM|\FILTER|\
  72 FIT|FITCURVE|FITNONLINEAR|FKEY|FLRV|FOR|FORECAST|FORMULA|FOURIER|\
  73 FPSEUDOFACTORS|FRAME|FSIMILARITY|FSSPM|FTSM|FVARIOGRAM|GENERATE|GET|\
  74 GETATTRIBUTE|GRAPH|GROUPS|HCLUSTER|HDISPLAY|HELP|HISTOGRAM|HLIST|\
  75 HSUMMARIZE|IF|INPUT|INTERPOLATE|IRREDUNDANT|JOB|KRIGE|LIST|LPCONTOUR|LPGRAPH|\
  76 LPHISTOGRAM|LRV|MARGIN|MATRIX|MCOVARIOGRAM|MDS|MERGE|MODEL|MONOTONIC|OPEN|\
  77 OPTION|OR|OUTPUT|OWN|PAGE|PARAMETER|PASS|PCO|PCP|PEN|POINTER|PREDICT|PRINT|\
  78 PROCEDURE|QUESTION|RANDOMIZE|RCYCLE|RDISPLAY|READ|RECORD|REDUCE|REFORMULATE|\
  79 RELATE|REML|RESTRICT|RESUME|RETRIEVE|RETURN|RFUNCTION|RKEEP|ROTATE|SCALAR|SET|\
  80 SETALLOCATIONS|SETCALCULATE|SETOPTION|SETPARAMETER|SETRELATE|SET2FORMULA|SKIP|\
  81 SORT|SPLOAD|SSPM|STEP|STOP|STORE|STRUCTURE|SUSPEND|SWITCH|SYMMETRICMATRIX|\
  82 TABLE|TABULATE|TDISPLAY|TERMS|TEXT|TKEEP|TRANSFERFUNCTION|TREATMENTSTRUCTURE|\
  83 TREE|TRY|TSM|TSUMMARIZE|TXBREAK|TXCONSTRUCT|TXFIND|TXPOSITION|UNITS|VARIATE|\
  84 VCOMPONENTS|VCYCLE|VDISPLAY|VKEEP|VPEDIGREE|VPREDICT|VRESIDUAL|VSTATUS|\
  85 VSTRUCTURE|XAXIS|YAXIS|ZAXIS'
  86 
  87     procedures = 'A2DISPLAY|A2KEEP|A2PLOT|A2WAY|ABIVARIATE|AFALPHA|\
  88 AFCARRYOVER|AFCYCLIC|AFIELDRESIDUALS|AFLABELS|AFORMS|AFUNITS|AGALPHA|AGBIB|\
  89 AGBOXBEHNKEN|AGCENTRALCOMPOSITE|AGCROSSOVERLATIN|AGCYCLIC|AGDESIGN|AGFRACTION|\
  90 AGHIERARCHICAL|AGLATIN|AGLOOP|AGMAINEFFECT|AGNATURALBLOCK|AGNEIGHBOUR|\
  91 AGQLATIN|AGRAPH|AGREFERENCE|AGSEMILATIN|AGSQLATTICE|AKAIKEHISTOGRAM|AKEY|\
  92 ALIAS|ALLPAIRWISE|AMERGE|AMMI|AMTDISPLAY|AMTIER|ANTMVESTIMATE|ANTORDER|\
  93 ANTTEST|AONEWAY|APERMTEST|APLOT|APOLYNOMIAL|APOWER|APPEND|APRODUCT|ARANDOMIZE|\
  94 AREPMEASURES|ASAMPLESIZE|ASCREEN|ASTATUS|ASWEEP|AUDISPLAY|AUKEEP|AUNBALANCED|\
  95 BANK|BARCHART|BCDISPLAY|BCIDENTIFY|BCLASSIFICATION|BCONSTRUCT|BCVALUES|BGRAPH|\
  96 BIPLOT|BJESTIMATE|BJFORECAST|BJIDENTIFY|BKDISPLAY|BKEY|BKIDENTIFY|BNTEST|\
  97 BOOTSTRAP|BOXPLOT|BPRINT|BPRUNE|BRDISPLAY|BREGRESSION|BRPREDICT|BRVALUES|\
  98 CANCORRELATION|CDESCRIBE|CENSOR|CHECKARGUMENT|CHISQUARE|CINTERACTION|CLASSIFY|\
  99 CONCORD|CONFIDENCE|CONVEXHULL|CORANALYSIS|CORRESP|COVDESIGN|CUMDISTRIBUTION|\
 100 CVAPLOT|CVASCORES|DAPLOT|DAYLENGTH|DBARCHART|DCIRCULAR|DCOMPOSITIONAL|\
 101 DDENDROGRAM|DDESIGN|DECIMALS|DEMC|DESCRIBE|DESIGN|DHELP|DIALLEL|DILUTION|\
 102 DISCRIMINATE|DKSTPLOT|DMASS|DMST|DOTPLOT|DPARALLEL|DPOLYGON|DPROBABILITY|\
 103 DPTMAP|DPTREAD|DREPMEASURES|DRESIDUALS|DRPOLYGON|DSCATTER|DTEXT|DTIMEPLOT|\
 104 ECABUNDANCEPLOT|ECANOSIM|ECDIVERSITY|ECFIT|ECNICHE|ECRAREFACTION|\
 105 EXTRABINOMIAL|F2DVARIOGRAM|FACAMEND|FACDIVIDE|FACPRODUCT|FACSORT|\
 106 FBASICCONTRASTS|FDESIGNFILE|FDRBONFERRONI|FDRMIXTURE|FEXACT2X2|FFRAME|\
 107 FHADAMARDMATRIX|FHAT|FIELLER|FILEREAD|FITINDIVIDUALLY|FITMULTIVARIATE|\
 108 FITNONNEGATIVE|FITPARALLEL|FITSCHNUTE|FLIBHELP|FMFACTORS|FPROJECTIONMATRIX|\
 109 FRESTRICTEDSET|FRIEDMAN|FSTRING|FTEXT|FVCOVARIANCE|FZERO|GALOIS|GEE|\
 110 GENPROCRUSTES|GHAT|GINVERSE|GLM|GLMM|GRANDOM|GRCSR|GRLABEL|GRMULTINORMAL|\
 111 GRTHIN|GRTORSHIFT|GSTATISTIC|HANOVA|HEATUNITS|HGANALYSE|HGDISPLAY|\
 112 HGDRANDOMMODEL|HGFIXEDMODEL|HGKEEP|HGPLOT|HGPREDICT|HGRANDOMMODEL|IDENTIFY|\
 113 IFUNCTION|INSIDE|JACKKNIFE|JOIN|K12HAT|KAPLANMEIER|KAPPA|KCONCORDANCE|\
 114 KCROSSVALIDATION|KCSRENVELOPES|KERNELDENSITY|KHAT|KLABENVELOPES|KRUSKAL|\
 115 KSED|KSTHAT|KSTMCTEST|KSTSE|KTAU|KTORENVELOPES|LCONCORDANCE|LIBEXAMPLE|\
 116 LIBFILENAME|LIBHELP|LIBINFORM|LIBMANUAL|LIBVERSION|LINDEPENDENCE|LORENZ|\
 117 LRVSCREE|LVARMODEL|MAANOVA|MANNWHITNEY|MANOVA|MANTEL|MCNEMAR|MEDIANTETRAD|\
 118 MPOWER|MSEKERNEL2D|MTABULATE|MULTMISSING|MVARIOGRAM|MVFILL|NCONVERT|NCSPLINE|\
 119 NLCONTRASTS|NORMTEST|NOTICE|ORTHPOLYNOMIAL|PAIRTEST|PARTIALCORRELATIONS|\
 120 PCOPROCRUSTES|PDESIGN|PDUPLICATE|PERCENT|PERIODTEST|PERMUTE|PLS|PNTEST|PPAIR|\
 121 PRCORRELATION|PREWHITEN|PRIMEPOWER|PRKTAU|PRMANNWHITNEYU|PROBITANALYSIS|\
 122 PRWILCOXON|PTAREAPOLYGON|PTBOX|PTCLOSEPOLYGON|PTDESCRIBE|PTGRID|PTINTENSITY|\
 123 PTK3D|PTKERNEL2D|PTREMOVE|PTSINSIDEPOLYGON|QCOCHRAN|QFACTOR|QLIST|QUANTILE|\
 124 R2LINES|RANK|RCHECK|RCOMPARISONS|REPLICATION|REPPERIODOGRAM|RGRAPH|RIDGE|\
 125 RJOINT|RLFUNCTIONAL|RLIFETABLE|RMGLM|RMULTIVARIATE|RNEGBINOMIAL|RNONNEGATIVE|\
 126 ROBSSPM|RPAIR|RPARALLEL|RPERMTEST|RPHCHANGE|RPHDISPLAY|RPHFIT|RPHKEEP|\
 127 RPHVECTORS|RPOWER|RPROPORTIONAL|RSCHNUTE|RSCREEN|RSEARCH|RSTEST|RSURVIVAL|\
 128 RUGPLOT|RUNTEST|RXGENSTAT|SAMPLE|SBNTEST|SCORRELATION|SED2ESE|SIGNTEST|\
 129 SIMPLEX|SKEWSYMMETRY|SLCONCORDANCE|SMANNWHITNEY|SMCNEMAR|SMOOTHSPECTRUM|\
 130 SPCAPABILITY|SPCCHART|SPCUSUM|SPEARMAN|SPEWMA|SPLINE|SPPCHART|SPRECISION|\
 131 SPSHEWHART|SSIGNTEST|STACK|STANDARDIZE|STEM|STTEST|SUBSET|SVBOOT|SVCALIBRATE|\
 132 SVGLM|SVREWEIGHT|SVSTRATIFIED|SVTABULATE|SVWEIGHT|TABMODE|TABSORT|TALLY|\
 133 TRELLIS|TTEST|UNSTACK|VEQUATE|VFUNCTION|VHOMOGENEITY|VINTERPOLATE|VMATRIX|\
 134 VORTHPOLYNOMIAL|VPLOT|VREGRESS|VTABLE|WADLEY|WILCOXON|WINDROSE|XOCATEGORIES|\
 135 XOEFFICIENCY|XOPOWER|YTRANSFORM'
 136 
 137     mswindows = 'DBEXPORT|DBIMPORT|DDEEXPORT|DDEIMPORT|EXPORT|IMPORT|%CD'
 138 
 139     operators = 'AND|EOR|EQ|EQS|GE|GT|IS|ISNT|IN|LE|LT|NE|NES|NI|NOT|OR'
 140 
 141     all_commands = set(declarations.split('|')+
 142                        flowcontrols.split('|')+
 143                        directives.split('|')+
 144                        procedures.split('|')+
 145                        mswindows.split('|'))
 146 
 147     functions ='ABS|ACOS|ANG|ANGLE|ANGULAR|ARCCOS|ARCSIN|ARCTAN|AREA|ASIN|\
 148 ATAN|BASE|BBELOW|BBRANCHES|BDEPTH|BETA|BI0|BI1|BJ0|BJ1|BMAXNODE|BNBRANCHES|\
 149 BNEXT|BNNODES|BOUND|BPATH|BPREVIOUS|BSCAN|BTERMINAL|C|CED|CEILING|CHARACTERS|\
 150 CHISQ|CHOLESKI|CIRCULATE|CLBETA|CLBINOMIAL|CLBVARIATENORMAL|CLCHISQUARE|CLF|\
 151 CLGAMMA|CLHYPERGEOMETRIC|CLINVNORMAL|CLLOGNORMAL|CLNORMAL|CLOGLOG|CLPOISSON|\
 152 CLSMMODULUS|CLSRANGE|CLT|CLUNIFORM|COLCENTRE|COLMEANS|COLNOBSERVATIONS|\
 153 COLSUMS|COL1|CONSTANTS|CORRELATION|CORRMAT|COS|COSH|COV|COVARIANCE|CPUTIME|\
 154 CUBETA|CUBINOMIAL|CUBVARIATENORMAL|CUCHISQUARE|CUF|CUGAMMA|CUHYPERGEOMETRIC|\
 155 CUINVNORMAL|CULOGNORMAL|CUM|CUMULATE|CUNORMAL|CUPOISSON|CUSMMODULUS|CUSRANGE|\
 156 CUT|CUUNIFORM|D|DATE|DAY|DEGREES|DET|DETERMINANT|DIAGONAL|DIFFERENCE|DIGAMMA|\
 157 DPRODUCT|DSUM|EDBETA|EDBINOMIAL|EDCHISQUARE|EDF|EDGAMMA|EDHYPERGEOMETRIC|\
 158 EDINVNORMAL|EDLOGNORMAL|EDNORMAL|EDPOISSON|EDSMMODULUS|EDSRANGE|EDT|EDUNIFORM|\
 159 ELEMENTS|EVALUES|EVECTORS|EXP|EXPAND|FACTORIAL|FED|FLOOR|FPROBABILITY|\
 160 FRACTION|FRATIO|GAMMA|GETFIRST|GETLAST|GETPOSITION|GINVERSE|GRBETA|GRBINOMIAL|\
 161 GRCHISQUARE|GRF|GRGAMMA|GRHYPERGEOMETRIC|GRLOGNORMAL|GRNORMAL|GRPOISSON|\
 162 GRSAMPLE|GRSELECT|GRT|GRUNIFORM|HOURS|I|IANGULAR|ICLOGLOG|IDENTITY|ILOGIT|INT|\
 163 INTEGER|INV|INVERSE|KRONECKER|KURTOSIS|LEAPYEAR|LLB|LLBINOMIAL|LLG|LLGAMMA|\
 164 LLN|LLNORMAL|LLP|LLPOISSON|LNFACTORIAL|LNGAMMA|LOG|LOG10|LOGIT|LSVECTORS|\
 165 LTPRODUCT|LTRIANGLE|MAT0|MAT1|MAX|MAXIMUM|MBASE|MCENTRE|MEAN|MED|MEDIAN|MEXP|\
 166 MFRACTION|MIN|MINIMUM|MINUTES|MODULO|MONTH|MPOWER|MSQRT|MVINSERT|MVREPLACE|\
 167 MZERO|NCOLUMNS|NCOMBINATIONS|NDAYINYEAR|NED|NEWLEVELS|NEXPAND|NLEVELS|NMV|\
 168 NOBSERVATIONS|NORMAL|NOW|NPERMUTATIONS|NROWS|NVALUES|NVRESTRICTED|\
 169 NVUNRESTRICTED|NWEEKINYEAR|PAREA|PERCENTILES|POSITION|PRBETA|PRBINOMIAL|\
 170 PRCHISQUARE|PRF|PRGAMMA|PRHYPERGEOMETRIC|PRINVNORMAL|PRLOGNORMAL|PRNORMAL|\
 171 PRODUCT|PRPOISSON|PRSMMODULUS|PRSRANGE|PRT|PRUNIFORM|QPRODUCT|QTPRODUCT|\
 172 QUANTILES|RADIANS|RANGE|RANKS|RESTRICTION|REVERSE|RMEANS|RNOBSERVATIONS|ROUND|\
 173 ROWCENTRE|ROWMEANS|ROWNOBSERVATIONS|ROWSUMS|ROW1|RSVECTORS|RTOTALS|RTPRODUCT|\
 174 RUNS|SD|SECONDS|SEMEAN|SET|SHIFT|SIGN|SIN|SINH|SKEWNESS|SOLUTION|SORT|SQRT|\
 175 SSPLINE|STANDARIZE|SUBMAT|SUM|SVALUES|T|TAN|TANH|TCOLUMN|TDIAGONAL|TIME|\
 176 TKURTOSIS|TMATRIX|TMAXIMA|TMEANS|TMEDIANS|TMINIMA|TNMV|TNOBSERVATIONS|\
 177 TNVALUES|TOTAL|TPROJECT|TRACE|TRANSPOSE|TRIGAMMA|TROW|TSD|TSEMEANS|TSKEWNESS|\
 178 TSUMS|TTOTALS|TVARIANCES|UNIQUE|UNSET|URAND|UTRIANGLE|VAR|VARIANCE|\
 179 VCORRELATION|VCOVARIANCE|VEC|VECH|VKURTOSIS|VMAXIMA|VMEANS|VMEDIANS|VMINIMA|\
 180 VNMV|VNOBSERVATIONS|VNVALUES|VPOSITIONS|VRANGE|VSD|VSEMEANS|VSKEWNESS|VSUMS|\
 181 VTOTALS|VVARIANCES|WEEKDAY|WHERE|YEAR'
 182 
 183     formula_functions = 'COMPARISON|LO|LOESS|POL|POLND|REG|REGND|S|SSPLINE'
 184 
 185     all_functions = set(functions.split('|') + 
 186                         formula_functions.split('|'))
 187 
 188     tokens = {
 189         'root': [
 190             (r'\n', Text.Whitespace),
 191 
 192             (r'"(""|[^"])*"', Comment), # standard comments are double-quoted
 193             
 194             (r"'", String, 'string'), # standard strings are single-quoted
 195             
 196             # continuation of statement to next line
 197             (r'(\\)(\s*\n)', bygroups(Punctuation, Text.Whitespace),
 198                 'continuation'),
 199 
 200             # text after continuation (an undocumented comment)
 201             (r'(\\)(.+\n)', bygroups(Punctuation, Comment), 'continuation'),
 202 
 203             # data termination (or redundant statement separator)
 204             (r':\s*\n', Punctuation),
 205 
 206             # function-like syntax
 207             (r'[!|\|][T|V][(]', Name), # undeclared structure, not a functiion
 208             (r'[!|\|][(]', Name), # undeclared structure, not a function
 209 
 210             # functions
 211             (r'(?=([a-zA-Z][a-zA-Z0-9]*[(]))', Text, 'function'),
 212 
 213             # commands
 214             # start on new line, follow a separator (:) or just repeat (&)
 215             (r'(^)(\s*)(?=[a-zA-Z%])', Text.Whitespace, 'command'),
 216             (r'(:)(\s*)(?=[a-zA-Z%])', bygroups(Punctuation, Text.Whitespace),
 217                 'command'),
 218             (r'(?=&)', Text.Whitespace, 'command'),
 219 
 220             # operators
 221             (r'[-+*/=<>]', Operator), # calculations and comparisons
 222             (r'(\.)(%s)(\.)' % operators, Operator.Word),
 223             (r'[$]', Operator), # subset of structure
 224             (r'[*]', Operator), # missing value in certain contexts
 225             (r'[#]', Operator), # substitution with content of structure
 226 
 227             # ellipsis - arithmetic progression
 228             (r'(\d+)(\.\.\.)(\d+)', bygroups(Number, Operator, Number)),
 229 
 230             (r'[\[\](){};,]', Punctuation),
 231 
 232             (r'[a-zA-Z_%][a-zA-Z_%0-9]*', Name),
 233 
 234             (r'(\d+\.?\d*|\d*\.\d+)([eE][+-]?[0-9]+)?', Number.Float),
 235             (r'\d+', Number.Integer),
 236             
 237             (r'.', Text), # anything else not captured
 238         ],
 239 
 240         'string': [
 241             # escaped back-slash in string
 242             (r'\\\\', String),
 243             # comment after continuation (undocumented)
 244             (r'(\\)(.*\n)', bygroups(Punctuation, Comment)),
 245             # string symbols
 246             (r'(~{)(~|alpha|beta|gamma|delta|epsilon|varepsilon|zeta|eta|theta|'
 247              r'vartheta|iota|kappa|lambda|mu|nu|xi|omicron|pi|varpi|rho|varrho|'
 248              r'sigma|varsigma|tau|upsilon|phi|varphi|chi|psi|omega|bull|bullet|'
 249              r'cdot|div|divide|gg|laquo|ll|middot|minus)(})',String.Symbol),
 250             # string markup
 251             (r'(~)(bold|b|italic|i|_|\^|html|h|latex|l|plain|p|rtf|r)({.*})',
 252                 String.Other),
 253             # other string content
 254             (r"''|[^']", String),
 255             # done
 256             (r"'", String, '#pop'),
 257             ],
 258 
 259         'command': [
 260             (r'(\s)*('+declarations+r')\b', Keyword.Declaration),
 261             (r'(\s)*('+flowcontrols+r')\b', Keyword),
 262             (r'(\s)*('+directives+r')\b', Keyword),
 263             (r'(\s)*('+procedures+r')\b', Keyword),
 264             (r'(\s)*('+mswindows+r')\b', Keyword),
 265             # unrecognised possible keyword at beginning of statement
 266             # this will be processed by get_tokens_unprocessed 
 267             (r'[a-zA-Z_%][a-zA-Z0-9_%]*', Keyword.Possible),
 268             (r'\b', Keyword, '#pop'),
 269             # repeated statement directive/procedure
 270             (r'&', Keyword, '#pop'),
 271             (r'.', Text) # anything else not captured
 272             ],
 273 
 274         'continuation': [
 275             (r'^(?=[a-zA-Z%])', Text.Continue, '#pop'),
 276             (r'^\s*', Text.Whitespace, '#pop'),
 277             ],
 278             
 279         'function': [
 280             (r'(' + functions + formula_functions + ')([(])', bygroups(\
 281                 Name.Function, Punctuation), '#pop'),
 282             # unrecognised possible function name
 283             # this will be processed by get_tokens_unprocessed 
 284             (r'([a-zA-Z][a-zA-Z0-9]*)([(])', bygroups(Name.Function.Pending,\
 285                 Punctuation), '#pop'),
 286             ],
 287     }
 288 
 289     def get_tokens_unprocessed(self, text):
 290         t_valid = False
 291         for index, token, value in \
 292                 RegexLexer.get_tokens_unprocessed(self, text):
 293             if token in [Keyword, Keyword.Declaration]:
 294                 if self.normalise:
 295                     value = value.upper()
 296                 if self.continued:
 297                     token = Text
 298                     self.continued = False
 299                 yield index, token, value
 300             elif token is Name.Function:
 301                 if self.normalise:
 302                     value = value.upper()
 303                 yield index, token, value
 304             elif token is Text.Continue:
 305                 self.continued = True
 306                 yield index, Text, value
 307             elif token is Keyword.Possible:
 308                 v = str(value).strip().upper()
 309                 if len(v) > 3:
 310                     for k in self.all_commands:
 311                         if k.startswith(v):
 312                             t_valid = True
 313                             break
 314                 if t_valid:
 315                     if self.normalise:
 316                         value = k # do this to un-abbreviate
 317                     yield index, Keyword, value
 318                     t_valid = False
 319                 else:
 320                     yield index, Name, value
 321             elif token is Name.Function.Pending:
 322                 v = str(value).strip().upper()
 323                 if len(v) > 3:
 324                     for k in self.all_functions:
 325                         if k.startswith(v):
 326                             t_valid = True
 327                             break
 328                 if t_valid:
 329                     if self.normalise:
 330                         value = k # do this to un-abbreviate
 331                     yield index, Name.Function, value
 332                     t_valid = False
 333                 else:
 334                     yield index, Name, value
 335             else:
 336                 yield index, token, value