// Copyright 2012 The Gorilla Authors. All rights reserved. // Use of this source code is governed by a BSD-style // license that can be found in the LICENSE file. package scanner import ( "fmt" "regexp" "strings" "unicode" "unicode/utf8" ) // tokenType identifies the type of lexical tokens. type tokenType int // String returns a string representation of the token type. func (t tokenType) String() string { return tokenNames[t] } // Token represents a token and the corresponding string. type Token struct { Type tokenType Value string Line int Column int } // String returns a string representation of the token. func (t *Token) String() string { if len(t.Value) > 10 { return fmt.Sprintf("%s (line: %d, column: %d): %.10q...", t.Type, t.Line, t.Column, t.Value) } return fmt.Sprintf("%s (line: %d, column: %d): %q", t.Type, t.Line, t.Column, t.Value) } // All tokens ----------------------------------------------------------------- // The complete list of tokens in CSS3. const ( // Scanner flags. TokenError tokenType = iota TokenEOF // From now on, only tokens from the CSS specification. TokenIdent TokenAtKeyword TokenString TokenHash TokenNumber TokenPercentage TokenDimension TokenURI TokenUnicodeRange TokenCDO TokenCDC TokenS TokenComment TokenFunction TokenIncludes TokenDashMatch TokenPrefixMatch TokenSuffixMatch TokenSubstringMatch TokenChar TokenBOM ) // tokenNames maps tokenType's to their names. Used for conversion to string. var tokenNames = map[tokenType]string{ TokenError: "error", TokenEOF: "EOF", TokenIdent: "IDENT", TokenAtKeyword: "ATKEYWORD", TokenString: "STRING", TokenHash: "HASH", TokenNumber: "NUMBER", TokenPercentage: "PERCENTAGE", TokenDimension: "DIMENSION", TokenURI: "URI", TokenUnicodeRange: "UNICODE-RANGE", TokenCDO: "CDO", TokenCDC: "CDC", TokenS: "S", TokenComment: "COMMENT", TokenFunction: "FUNCTION", TokenIncludes: "INCLUDES", TokenDashMatch: "DASHMATCH", TokenPrefixMatch: "PREFIXMATCH", TokenSuffixMatch: "SUFFIXMATCH", TokenSubstringMatch: "SUBSTRINGMATCH", TokenChar: "CHAR", TokenBOM: "BOM", } // Macros and productions ----------------------------------------------------- // http://www.w3.org/TR/css3-syntax/#tokenization var macroRegexp = regexp.MustCompile(`\{[a-z]+\}`) // macros maps macro names to patterns to be expanded. var macros = map[string]string{ // must be escaped: `\.+*?()|[]{}^$` "ident": `-?{nmstart}{nmchar}*`, "name": `{nmchar}+`, "nmstart": `[a-zA-Z_]|{nonascii}|{escape}`, "nonascii": "[\u0080-\uD7FF\uE000-\uFFFD\U00010000-\U0010FFFF]", "unicode": `\\[0-9a-fA-F]{1,6}{wc}?`, "escape": "{unicode}|\\\\[\u0020-\u007E\u0080-\uD7FF\uE000-\uFFFD\U00010000-\U0010FFFF]", "nmchar": `[a-zA-Z0-9_-]|{nonascii}|{escape}`, "num": `[0-9]*\.[0-9]+|[0-9]+`, "string": `"(?:{stringchar}|')*"|'(?:{stringchar}|")*'`, "stringchar": `{urlchar}|[ ]|\\{nl}`, "nl": `[\n\r\f]|\r\n`, "w": `{wc}*`, "wc": `[\t\n\f\r ]`, // urlchar should accept [(ascii characters minus those that need escaping)|{nonascii}|{escape}] // ASCII characters range = `[\u0020-\u007e]` // Skip space \u0020 = `[\u0021-\u007e]` // Skip quotation mark \0022 = `[\u0021\u0023-\u007e]` // Skip apostrophe \u0027 = `[\u0021\u0023-\u0026\u0028-\u007e]` // Skip reverse solidus \u005c = `[\u0021\u0023-\u0026\u0028-\u005b\u005d\u007e]` // Finally, the left square bracket (\u005b) and right (\u005d) needs escaping themselves "urlchar": "[\u0021\u0023-\u0026\u0028-\\\u005b\\\u005d-\u007E]|{nonascii}|{escape}", } // productions maps the list of tokens to patterns to be expanded. var productions = map[tokenType]string{ // Unused regexps (matched using other methods) are commented out. TokenIdent: `{ident}`, TokenAtKeyword: `@{ident}`, TokenString: `{string}`, TokenHash: `#{name}`, TokenNumber: `{num}`, TokenPercentage: `{num}%`, TokenDimension: `{num}{ident}`, TokenURI: `url\({w}(?:{string}|{urlchar}*?){w}\)`, TokenUnicodeRange: `U\+[0-9A-F\?]{1,6}(?:-[0-9A-F]{1,6})?`, //TokenCDO: `<!--`, TokenCDC: `-->`, TokenS: `{wc}+`, TokenComment: `/\*[^\*]*[\*]+(?:[^/][^\*]*[\*]+)*/`, TokenFunction: `{ident}\(`, //TokenIncludes: `~=`, //TokenDashMatch: `\|=`, //TokenPrefixMatch: `\^=`, //TokenSuffixMatch: `\$=`, //TokenSubstringMatch: `\*=`, //TokenChar: `[^"']`, //TokenBOM: "\uFEFF", } // matchers maps the list of tokens to compiled regular expressions. // // The map is filled on init() using the macros and productions defined in // the CSS specification. var matchers = map[tokenType]*regexp.Regexp{} // matchOrder is the order to test regexps when first-char shortcuts // can't be used. var matchOrder = []tokenType{ TokenURI, TokenFunction, TokenUnicodeRange, TokenIdent, TokenDimension, TokenPercentage, TokenNumber, TokenCDC, } func init() { // replace macros and compile regexps for productions. replaceMacro := func(s string) string { return "(?:" + macros[s[1:len(s)-1]] + ")" } for t, s := range productions { for macroRegexp.MatchString(s) { s = macroRegexp.ReplaceAllStringFunc(s, replaceMacro) } matchers[t] = regexp.MustCompile("^(?:" + s + ")") } } // Scanner -------------------------------------------------------------------- // New returns a new CSS scanner for the given input. func New(input string) *Scanner { // Normalize newlines. input = strings.Replace(input, "\r\n", "\n", -1) return &Scanner{ input: input, row: 1, col: 1, } } // Scanner scans an input and emits tokens following the CSS3 specification. type Scanner struct { input string pos int row int col int err *Token } // Next returns the next token from the input. // // At the end of the input the token type is TokenEOF. // // If the input can't be tokenized the token type is TokenError. This occurs // in case of unclosed quotation marks or comments. func (s *Scanner) Next() *Token { if s.err != nil { return s.err } if s.pos >= len(s.input) { s.err = &Token{TokenEOF, "", s.row, s.col} return s.err } if s.pos == 0 { // Test BOM only once, at the beginning of the file. if strings.HasPrefix(s.input, "\uFEFF") { return s.emitSimple(TokenBOM, "\uFEFF") } } // There's a lot we can guess based on the first byte so we'll take a // shortcut before testing multiple regexps. input := s.input[s.pos:] switch input[0] { case '\t', '\n', '\f', '\r', ' ': // Whitespace. return s.emitToken(TokenS, matchers[TokenS].FindString(input)) case '.': // Dot is too common to not have a quick check. // We'll test if this is a Char; if it is followed by a number it is a // dimension/percentage/number, and this will be matched later. if len(input) > 1 && !unicode.IsDigit(rune(input[1])) { return s.emitSimple(TokenChar, ".") } case '#': // Another common one: Hash or Char. if match := matchers[TokenHash].FindString(input); match != "" { return s.emitToken(TokenHash, match) } return s.emitSimple(TokenChar, "#") case '@': // Another common one: AtKeyword or Char. if match := matchers[TokenAtKeyword].FindString(input); match != "" { return s.emitSimple(TokenAtKeyword, match) } return s.emitSimple(TokenChar, "@") case ':', ',', ';', '%', '&', '+', '=', '>', '(', ')', '[', ']', '{', '}': // More common chars. return s.emitSimple(TokenChar, string(input[0])) case '"', '\'': // String or error. match := matchers[TokenString].FindString(input) if match != "" { return s.emitToken(TokenString, match) } s.err = &Token{TokenError, "unclosed quotation mark", s.row, s.col} return s.err case '/': // Comment, error or Char. if len(input) > 1 && input[1] == '*' { match := matchers[TokenComment].FindString(input) if match != "" { return s.emitToken(TokenComment, match) } else { s.err = &Token{TokenError, "unclosed comment", s.row, s.col} return s.err } } return s.emitSimple(TokenChar, "/") case '~': // Includes or Char. return s.emitPrefixOrChar(TokenIncludes, "~=") case '|': // DashMatch or Char. return s.emitPrefixOrChar(TokenDashMatch, "|=") case '^': // PrefixMatch or Char. return s.emitPrefixOrChar(TokenPrefixMatch, "^=") case '$': // SuffixMatch or Char. return s.emitPrefixOrChar(TokenSuffixMatch, "$=") case '*': // SubstringMatch or Char. return s.emitPrefixOrChar(TokenSubstringMatch, "*=") case '<': // CDO or Char. return s.emitPrefixOrChar(TokenCDO, "<!--") } // Test all regexps, in order. for _, token := range matchOrder { if match := matchers[token].FindString(input); match != "" { return s.emitToken(token, match) } } // We already handled unclosed quotation marks and comments, // so this can only be a Char. r, width := utf8.DecodeRuneInString(input) token := &Token{TokenChar, string(r), s.row, s.col} s.col += width s.pos += width return token } // updatePosition updates input coordinates based on the consumed text. func (s *Scanner) updatePosition(text string) { width := utf8.RuneCountInString(text) lines := strings.Count(text, "\n") s.row += lines if lines == 0 { s.col += width } else { s.col = utf8.RuneCountInString(text[strings.LastIndex(text, "\n"):]) } s.pos += len(text) // while col is a rune index, pos is a byte index } // emitToken returns a Token for the string v and updates the scanner position. func (s *Scanner) emitToken(t tokenType, v string) *Token { token := &Token{t, v, s.row, s.col} s.updatePosition(v) return token } // emitSimple returns a Token for the string v and updates the scanner // position in a simplified manner. // // The string is known to have only ASCII characters and to not have a newline. func (s *Scanner) emitSimple(t tokenType, v string) *Token { token := &Token{t, v, s.row, s.col} s.col += len(v) s.pos += len(v) return token } // emitPrefixOrChar returns a Token for type t if the current position // matches the given prefix. Otherwise it returns a Char token using the // first character from the prefix. // // The prefix is known to have only ASCII characters and to not have a newline. func (s *Scanner) emitPrefixOrChar(t tokenType, prefix string) *Token { if strings.HasPrefix(s.input[s.pos:], prefix) { return s.emitSimple(t, prefix) } return s.emitSimple(TokenChar, string(prefix[0])) }