http://git-wip-us.apache.org/repos/asf/incubator-mynewt-newt/blob/4ae9684f/newt/Godeps/_workspace/src/github.com/hashicorp/hcl/hcl/parser/parser.go ---------------------------------------------------------------------- diff --git a/newt/Godeps/_workspace/src/github.com/hashicorp/hcl/hcl/parser/parser.go b/newt/Godeps/_workspace/src/github.com/hashicorp/hcl/hcl/parser/parser.go deleted file mode 100644 index 086c087..0000000 --- a/newt/Godeps/_workspace/src/github.com/hashicorp/hcl/hcl/parser/parser.go +++ /dev/null @@ -1,422 +0,0 @@ -// Package parser implements a parser for HCL (HashiCorp Configuration -// Language) -package parser - -import ( - "errors" - "fmt" - - "github.com/hashicorp/hcl/hcl/ast" - "github.com/hashicorp/hcl/hcl/scanner" - "github.com/hashicorp/hcl/hcl/token" -) - -type Parser struct { - sc *scanner.Scanner - - // Last read token - tok token.Token - commaPrev token.Token - - comments []*ast.CommentGroup - leadComment *ast.CommentGroup // last lead comment - lineComment *ast.CommentGroup // last line comment - - enableTrace bool - indent int - n int // buffer size (max = 1) -} - -func newParser(src []byte) *Parser { - return &Parser{ - sc: scanner.New(src), - } -} - -// Parse returns the fully parsed source and returns the abstract syntax tree. -func Parse(src []byte) (*ast.File, error) { - p := newParser(src) - return p.Parse() -} - -var errEofToken = errors.New("EOF token found") - -// Parse returns the fully parsed source and returns the abstract syntax tree. -func (p *Parser) Parse() (*ast.File, error) { - f := &ast.File{} - var err, scerr error - p.sc.Error = func(pos token.Pos, msg string) { - scerr = &PosError{Pos: pos, Err: errors.New(msg)} - } - - f.Node, err = p.objectList() - if scerr != nil { - return nil, scerr - } - if err != nil { - return nil, err - } - - f.Comments = p.comments - return f, nil -} - -func (p *Parser) objectList() (*ast.ObjectList, error) { - defer un(trace(p, "ParseObjectList")) - node := &ast.ObjectList{} - - for { - n, err := p.objectItem() - if err == errEofToken { - break // we are finished - } - - // we don't return a nil node, because might want to use already - // collected items. - if err != nil { - return node, err - } - - node.Add(n) - } - return node, nil -} - -func (p *Parser) consumeComment() (comment *ast.Comment, endline int) { - endline = p.tok.Pos.Line - - // count the endline if it's multiline comment, ie starting with /* - if len(p.tok.Text) > 1 && p.tok.Text[1] == '*' { - // don't use range here - no need to decode Unicode code points - for i := 0; i < len(p.tok.Text); i++ { - if p.tok.Text[i] == '\n' { - endline++ - } - } - } - - comment = &ast.Comment{Start: p.tok.Pos, Text: p.tok.Text} - p.tok = p.sc.Scan() - return -} - -func (p *Parser) consumeCommentGroup(n int) (comments *ast.CommentGroup, endline int) { - var list []*ast.Comment - endline = p.tok.Pos.Line - - for p.tok.Type == token.COMMENT && p.tok.Pos.Line <= endline+n { - var comment *ast.Comment - comment, endline = p.consumeComment() - list = append(list, comment) - } - - // add comment group to the comments list - comments = &ast.CommentGroup{List: list} - p.comments = append(p.comments, comments) - - return -} - -// objectItem parses a single object item -func (p *Parser) objectItem() (*ast.ObjectItem, error) { - defer un(trace(p, "ParseObjectItem")) - - keys, err := p.objectKey() - if err != nil { - return nil, err - } - - o := &ast.ObjectItem{ - Keys: keys, - } - - if p.leadComment != nil { - o.LeadComment = p.leadComment - p.leadComment = nil - } - - switch p.tok.Type { - case token.ASSIGN: - o.Assign = p.tok.Pos - o.Val, err = p.object() - if err != nil { - return nil, err - } - case token.LBRACE: - o.Val, err = p.objectType() - if err != nil { - return nil, err - } - } - - // do a look-ahead for line comment - p.scan() - if len(keys) > 0 && o.Val.Pos().Line == keys[0].Pos().Line && p.lineComment != nil { - o.LineComment = p.lineComment - p.lineComment = nil - } - p.unscan() - return o, nil -} - -// objectKey parses an object key and returns a ObjectKey AST -func (p *Parser) objectKey() ([]*ast.ObjectKey, error) { - keyCount := 0 - keys := make([]*ast.ObjectKey, 0) - - for { - tok := p.scan() - switch tok.Type { - case token.EOF: - return nil, errEofToken - case token.ASSIGN: - // assignment or object only, but not nested objects. this is not - // allowed: `foo bar = {}` - if keyCount > 1 { - return nil, &PosError{ - Pos: p.tok.Pos, - Err: fmt.Errorf("nested object expected: LBRACE got: %s", p.tok.Type), - } - } - - if keyCount == 0 { - return nil, &PosError{ - Pos: p.tok.Pos, - Err: errors.New("no object keys found!"), - } - } - - return keys, nil - case token.LBRACE: - // object - return keys, nil - case token.IDENT, token.STRING: - keyCount++ - keys = append(keys, &ast.ObjectKey{Token: p.tok}) - case token.ILLEGAL: - fmt.Println("illegal") - default: - return nil, &PosError{ - Pos: p.tok.Pos, - Err: fmt.Errorf("expected: IDENT | STRING | ASSIGN | LBRACE got: %s", p.tok.Type), - } - } - } -} - -// object parses any type of object, such as number, bool, string, object or -// list. -func (p *Parser) object() (ast.Node, error) { - defer un(trace(p, "ParseType")) - tok := p.scan() - - switch tok.Type { - case token.NUMBER, token.FLOAT, token.BOOL, token.STRING, token.HEREDOC: - return p.literalType() - case token.LBRACE: - return p.objectType() - case token.LBRACK: - return p.listType() - case token.COMMENT: - // implement comment - case token.EOF: - return nil, errEofToken - } - - return nil, &PosError{ - Pos: tok.Pos, - Err: fmt.Errorf("Unknown token: %+v", tok), - } -} - -// objectType parses an object type and returns a ObjectType AST -func (p *Parser) objectType() (*ast.ObjectType, error) { - defer un(trace(p, "ParseObjectType")) - - // we assume that the currently scanned token is a LBRACE - o := &ast.ObjectType{ - Lbrace: p.tok.Pos, - } - - l, err := p.objectList() - - // if we hit RBRACE, we are good to go (means we parsed all Items), if it's - // not a RBRACE, it's an syntax error and we just return it. - if err != nil && p.tok.Type != token.RBRACE { - return nil, err - } - - // If there is no error, we should be at a RBRACE to end the object - if p.tok.Type != token.RBRACE { - return nil, fmt.Errorf("object expected closing RBRACE got: %s", p.tok.Type) - } - - o.List = l - o.Rbrace = p.tok.Pos // advanced via parseObjectList - return o, nil -} - -// listType parses a list type and returns a ListType AST -func (p *Parser) listType() (*ast.ListType, error) { - defer un(trace(p, "ParseListType")) - - // we assume that the currently scanned token is a LBRACK - l := &ast.ListType{ - Lbrack: p.tok.Pos, - } - - needComma := false - for { - tok := p.scan() - switch tok.Type { - case token.NUMBER, token.FLOAT, token.STRING, token.HEREDOC: - if needComma { - return nil, &PosError{ - Pos: tok.Pos, - Err: fmt.Errorf("unexpected token: %s. Expecting %s", tok.Type, token.COMMA), - } - } - - node, err := p.literalType() - if err != nil { - return nil, err - } - - l.Add(node) - needComma = true - case token.COMMA: - // get next list item or we are at the end - // do a look-ahead for line comment - p.scan() - if p.lineComment != nil { - lit, ok := l.List[len(l.List)-1].(*ast.LiteralType) - if ok { - lit.LineComment = p.lineComment - l.List[len(l.List)-1] = lit - p.lineComment = nil - } - } - p.unscan() - - needComma = false - continue - case token.BOOL: - // TODO(arslan) should we support? not supported by HCL yet - case token.LBRACK: - // TODO(arslan) should we support nested lists? Even though it's - // written in README of HCL, it's not a part of the grammar - // (not defined in parse.y) - case token.RBRACK: - // finished - l.Rbrack = p.tok.Pos - return l, nil - default: - return nil, &PosError{ - Pos: tok.Pos, - Err: fmt.Errorf("unexpected token while parsing list: %s", tok.Type), - } - } - } -} - -// literalType parses a literal type and returns a LiteralType AST -func (p *Parser) literalType() (*ast.LiteralType, error) { - defer un(trace(p, "ParseLiteral")) - - return &ast.LiteralType{ - Token: p.tok, - }, nil -} - -// scan returns the next token from the underlying scanner. If a token has -// been unscanned then read that instead. In the process, it collects any -// comment groups encountered, and remembers the last lead and line comments. -func (p *Parser) scan() token.Token { - // If we have a token on the buffer, then return it. - if p.n != 0 { - p.n = 0 - return p.tok - } - - // Otherwise read the next token from the scanner and Save it to the buffer - // in case we unscan later. - prev := p.tok - p.tok = p.sc.Scan() - - if p.tok.Type == token.COMMENT { - var comment *ast.CommentGroup - var endline int - - // fmt.Printf("p.tok.Pos.Line = %+v prev: %d endline %d \n", - // p.tok.Pos.Line, prev.Pos.Line, endline) - if p.tok.Pos.Line == prev.Pos.Line { - // The comment is on same line as the previous token; it - // cannot be a lead comment but may be a line comment. - comment, endline = p.consumeCommentGroup(0) - if p.tok.Pos.Line != endline { - // The next token is on a different line, thus - // the last comment group is a line comment. - p.lineComment = comment - } - } - - // consume successor comments, if any - endline = -1 - for p.tok.Type == token.COMMENT { - comment, endline = p.consumeCommentGroup(1) - } - - if endline+1 == p.tok.Pos.Line && p.tok.Type != token.RBRACE { - switch p.tok.Type { - case token.RBRACE, token.RBRACK: - // Do not count for these cases - default: - // The next token is following on the line immediately after the - // comment group, thus the last comment group is a lead comment. - p.leadComment = comment - } - } - - } - - return p.tok -} - -// unscan pushes the previously read token back onto the buffer. -func (p *Parser) unscan() { - p.n = 1 -} - -// ---------------------------------------------------------------------------- -// Parsing support - -func (p *Parser) printTrace(a ...interface{}) { - if !p.enableTrace { - return - } - - const dots = ". . . . . . . . . . . . . . . . . . . . . . . . . . . . . . . . " - const n = len(dots) - fmt.Printf("%5d:%3d: ", p.tok.Pos.Line, p.tok.Pos.Column) - - i := 2 * p.indent - for i > n { - fmt.Print(dots) - i -= n - } - // i <= n - fmt.Print(dots[0:i]) - fmt.Println(a...) -} - -func trace(p *Parser, msg string) *Parser { - p.printTrace(msg, "(") - p.indent++ - return p -} - -// Usage pattern: defer un(trace(p, "...")) -func un(p *Parser) { - p.indent-- - p.printTrace(")") -}
http://git-wip-us.apache.org/repos/asf/incubator-mynewt-newt/blob/4ae9684f/newt/Godeps/_workspace/src/github.com/hashicorp/hcl/hcl/parser/parser_test.go ---------------------------------------------------------------------- diff --git a/newt/Godeps/_workspace/src/github.com/hashicorp/hcl/hcl/parser/parser_test.go b/newt/Godeps/_workspace/src/github.com/hashicorp/hcl/hcl/parser/parser_test.go deleted file mode 100644 index ffe6edd..0000000 --- a/newt/Godeps/_workspace/src/github.com/hashicorp/hcl/hcl/parser/parser_test.go +++ /dev/null @@ -1,334 +0,0 @@ -package parser - -import ( - "fmt" - "io/ioutil" - "path/filepath" - "reflect" - "runtime" - "testing" - - "github.com/hashicorp/hcl/hcl/ast" - "github.com/hashicorp/hcl/hcl/token" -) - -func TestType(t *testing.T) { - var literals = []struct { - typ token.Type - src string - }{ - {token.STRING, `foo = "foo"`}, - {token.NUMBER, `foo = 123`}, - {token.NUMBER, `foo = -29`}, - {token.FLOAT, `foo = 123.12`}, - {token.FLOAT, `foo = -123.12`}, - {token.BOOL, `foo = true`}, - {token.HEREDOC, "foo = <<EOF\nHello\nWorld\nEOF"}, - } - - for _, l := range literals { - p := newParser([]byte(l.src)) - item, err := p.objectItem() - if err != nil { - t.Error(err) - } - - lit, ok := item.Val.(*ast.LiteralType) - if !ok { - t.Errorf("node should be of type LiteralType, got: %T", item.Val) - } - - if lit.Token.Type != l.typ { - t.Errorf("want: %s, got: %s", l.typ, lit.Token.Type) - } - } -} - -func TestListType(t *testing.T) { - var literals = []struct { - src string - tokens []token.Type - }{ - { - `foo = ["123", 123]`, - []token.Type{token.STRING, token.NUMBER}, - }, - { - `foo = [123, "123",]`, - []token.Type{token.NUMBER, token.STRING}, - }, - { - `foo = []`, - []token.Type{}, - }, - { - `foo = ["123", 123]`, - []token.Type{token.STRING, token.NUMBER}, - }, - { - `foo = [1, -"string", -<<EOF -heredoc contents -EOF -]`, - []token.Type{token.NUMBER, token.STRING, token.HEREDOC}, - }, - } - - for _, l := range literals { - p := newParser([]byte(l.src)) - item, err := p.objectItem() - if err != nil { - t.Error(err) - } - - list, ok := item.Val.(*ast.ListType) - if !ok { - t.Errorf("node should be of type LiteralType, got: %T", item.Val) - } - - tokens := []token.Type{} - for _, li := range list.List { - if tp, ok := li.(*ast.LiteralType); ok { - tokens = append(tokens, tp.Token.Type) - } - } - - equals(t, l.tokens, tokens) - } -} - -func TestObjectType(t *testing.T) { - var literals = []struct { - src string - nodeType []ast.Node - itemLen int - }{ - { - `foo = {}`, - nil, - 0, - }, - { - `foo = { - bar = "fatih" - }`, - []ast.Node{&ast.LiteralType{}}, - 1, - }, - { - `foo = { - bar = "fatih" - baz = ["arslan"] - }`, - []ast.Node{ - &ast.LiteralType{}, - &ast.ListType{}, - }, - 2, - }, - { - `foo = { - bar {} - }`, - []ast.Node{ - &ast.ObjectType{}, - }, - 1, - }, - { - `foo { - bar {} - foo = true - }`, - []ast.Node{ - &ast.ObjectType{}, - &ast.LiteralType{}, - }, - 2, - }, - } - - for _, l := range literals { - p := newParser([]byte(l.src)) - // p.enableTrace = true - item, err := p.objectItem() - if err != nil { - t.Error(err) - continue - } - - // we know that the ObjectKey name is foo for all cases, what matters - // is the object - obj, ok := item.Val.(*ast.ObjectType) - if !ok { - t.Errorf("node should be of type LiteralType, got: %T", item.Val) - continue - } - - // check if the total length of items are correct - equals(t, l.itemLen, len(obj.List.Items)) - - // check if the types are correct - for i, item := range obj.List.Items { - equals(t, reflect.TypeOf(l.nodeType[i]), reflect.TypeOf(item.Val)) - } - } -} - -func TestObjectKey(t *testing.T) { - keys := []struct { - exp []token.Type - src string - }{ - {[]token.Type{token.IDENT}, `foo {}`}, - {[]token.Type{token.IDENT}, `foo = {}`}, - {[]token.Type{token.IDENT}, `foo = bar`}, - {[]token.Type{token.IDENT}, `foo = 123`}, - {[]token.Type{token.IDENT}, `foo = "${var.bar}`}, - {[]token.Type{token.STRING}, `"foo" {}`}, - {[]token.Type{token.STRING}, `"foo" = {}`}, - {[]token.Type{token.STRING}, `"foo" = "${var.bar}`}, - {[]token.Type{token.IDENT, token.IDENT}, `foo bar {}`}, - {[]token.Type{token.IDENT, token.STRING}, `foo "bar" {}`}, - {[]token.Type{token.STRING, token.IDENT}, `"foo" bar {}`}, - {[]token.Type{token.IDENT, token.IDENT, token.IDENT}, `foo bar baz {}`}, - } - - for _, k := range keys { - p := newParser([]byte(k.src)) - keys, err := p.objectKey() - if err != nil { - t.Fatal(err) - } - - tokens := []token.Type{} - for _, o := range keys { - tokens = append(tokens, o.Token.Type) - } - - equals(t, k.exp, tokens) - } - - errKeys := []struct { - src string - }{ - {`foo 12 {}`}, - {`foo bar = {}`}, - {`foo []`}, - {`12 {}`}, - } - - for _, k := range errKeys { - p := newParser([]byte(k.src)) - _, err := p.objectKey() - if err == nil { - t.Errorf("case '%s' should give an error", k.src) - } - } -} - -// Official HCL tests -func TestParse(t *testing.T) { - cases := []struct { - Name string - Err bool - }{ - { - "assign_colon.hcl", - true, - }, - { - "comment.hcl", - false, - }, - { - "comment_lastline.hcl", - false, - }, - { - "comment_single.hcl", - false, - }, - { - "empty.hcl", - false, - }, - { - "list_comma.hcl", - false, - }, - { - "multiple.hcl", - false, - }, - { - "structure.hcl", - false, - }, - { - "structure_basic.hcl", - false, - }, - { - "structure_empty.hcl", - false, - }, - { - "complex.hcl", - false, - }, - { - "assign_deep.hcl", - true, - }, - { - "types.hcl", - false, - }, - { - "array_comment.hcl", - false, - }, - { - "array_comment_2.hcl", - true, - }, - { - "missing_braces.hcl", - true, - }, - { - "unterminated_object.hcl", - true, - }, - { - "unterminated_object_2.hcl", - true, - }, - } - - const fixtureDir = "./test-fixtures" - - for _, tc := range cases { - d, err := ioutil.ReadFile(filepath.Join(fixtureDir, tc.Name)) - if err != nil { - t.Fatalf("err: %s", err) - } - - _, err = Parse(d) - if (err != nil) != tc.Err { - t.Fatalf("Input: %s\n\nError: %s", tc.Name, err) - } - } -} - -// equals fails the test if exp is not equal to act. -func equals(tb testing.TB, exp, act interface{}) { - if !reflect.DeepEqual(exp, act) { - _, file, line, _ := runtime.Caller(1) - fmt.Printf("\033[31m%s:%d:\n\n\texp: %#v\n\n\tgot: %#v\033[39m\n\n", filepath.Base(file), line, exp, act) - tb.FailNow() - } -} http://git-wip-us.apache.org/repos/asf/incubator-mynewt-newt/blob/4ae9684f/newt/Godeps/_workspace/src/github.com/hashicorp/hcl/hcl/parser/test-fixtures/array_comment.hcl ---------------------------------------------------------------------- diff --git a/newt/Godeps/_workspace/src/github.com/hashicorp/hcl/hcl/parser/test-fixtures/array_comment.hcl b/newt/Godeps/_workspace/src/github.com/hashicorp/hcl/hcl/parser/test-fixtures/array_comment.hcl deleted file mode 100644 index 78c2675..0000000 --- a/newt/Godeps/_workspace/src/github.com/hashicorp/hcl/hcl/parser/test-fixtures/array_comment.hcl +++ /dev/null @@ -1,4 +0,0 @@ -foo = [ - "1", - "2", # comment -] http://git-wip-us.apache.org/repos/asf/incubator-mynewt-newt/blob/4ae9684f/newt/Godeps/_workspace/src/github.com/hashicorp/hcl/hcl/parser/test-fixtures/array_comment_2.hcl ---------------------------------------------------------------------- diff --git a/newt/Godeps/_workspace/src/github.com/hashicorp/hcl/hcl/parser/test-fixtures/array_comment_2.hcl b/newt/Godeps/_workspace/src/github.com/hashicorp/hcl/hcl/parser/test-fixtures/array_comment_2.hcl deleted file mode 100644 index f916677..0000000 --- a/newt/Godeps/_workspace/src/github.com/hashicorp/hcl/hcl/parser/test-fixtures/array_comment_2.hcl +++ /dev/null @@ -1,6 +0,0 @@ -provisioner "remote-exec" { - scripts = [ - "${path.module}/scripts/install-consul.sh" // missing comma - "${path.module}/scripts/install-haproxy.sh" - ] -} http://git-wip-us.apache.org/repos/asf/incubator-mynewt-newt/blob/4ae9684f/newt/Godeps/_workspace/src/github.com/hashicorp/hcl/hcl/parser/test-fixtures/assign_colon.hcl ---------------------------------------------------------------------- diff --git a/newt/Godeps/_workspace/src/github.com/hashicorp/hcl/hcl/parser/test-fixtures/assign_colon.hcl b/newt/Godeps/_workspace/src/github.com/hashicorp/hcl/hcl/parser/test-fixtures/assign_colon.hcl deleted file mode 100644 index eb5a99a..0000000 --- a/newt/Godeps/_workspace/src/github.com/hashicorp/hcl/hcl/parser/test-fixtures/assign_colon.hcl +++ /dev/null @@ -1,6 +0,0 @@ -resource = [{ - "foo": { - "bar": {}, - "baz": [1, 2, "foo"], - } -}] http://git-wip-us.apache.org/repos/asf/incubator-mynewt-newt/blob/4ae9684f/newt/Godeps/_workspace/src/github.com/hashicorp/hcl/hcl/parser/test-fixtures/assign_deep.hcl ---------------------------------------------------------------------- diff --git a/newt/Godeps/_workspace/src/github.com/hashicorp/hcl/hcl/parser/test-fixtures/assign_deep.hcl b/newt/Godeps/_workspace/src/github.com/hashicorp/hcl/hcl/parser/test-fixtures/assign_deep.hcl deleted file mode 100644 index dd3151c..0000000 --- a/newt/Godeps/_workspace/src/github.com/hashicorp/hcl/hcl/parser/test-fixtures/assign_deep.hcl +++ /dev/null @@ -1,5 +0,0 @@ -resource = [{ - foo = [{ - bar = {} - }] -}] http://git-wip-us.apache.org/repos/asf/incubator-mynewt-newt/blob/4ae9684f/newt/Godeps/_workspace/src/github.com/hashicorp/hcl/hcl/parser/test-fixtures/comment.hcl ---------------------------------------------------------------------- diff --git a/newt/Godeps/_workspace/src/github.com/hashicorp/hcl/hcl/parser/test-fixtures/comment.hcl b/newt/Godeps/_workspace/src/github.com/hashicorp/hcl/hcl/parser/test-fixtures/comment.hcl deleted file mode 100644 index 1ff7f29..0000000 --- a/newt/Godeps/_workspace/src/github.com/hashicorp/hcl/hcl/parser/test-fixtures/comment.hcl +++ /dev/null @@ -1,15 +0,0 @@ -// Foo - -/* Bar */ - -/* -/* -Baz -*/ - -# Another - -# Multiple -# Lines - -foo = "bar" http://git-wip-us.apache.org/repos/asf/incubator-mynewt-newt/blob/4ae9684f/newt/Godeps/_workspace/src/github.com/hashicorp/hcl/hcl/parser/test-fixtures/comment_lastline.hcl ---------------------------------------------------------------------- diff --git a/newt/Godeps/_workspace/src/github.com/hashicorp/hcl/hcl/parser/test-fixtures/comment_lastline.hcl b/newt/Godeps/_workspace/src/github.com/hashicorp/hcl/hcl/parser/test-fixtures/comment_lastline.hcl deleted file mode 100644 index 5529b9b..0000000 --- a/newt/Godeps/_workspace/src/github.com/hashicorp/hcl/hcl/parser/test-fixtures/comment_lastline.hcl +++ /dev/null @@ -1 +0,0 @@ -#foo \ No newline at end of file http://git-wip-us.apache.org/repos/asf/incubator-mynewt-newt/blob/4ae9684f/newt/Godeps/_workspace/src/github.com/hashicorp/hcl/hcl/parser/test-fixtures/comment_single.hcl ---------------------------------------------------------------------- diff --git a/newt/Godeps/_workspace/src/github.com/hashicorp/hcl/hcl/parser/test-fixtures/comment_single.hcl b/newt/Godeps/_workspace/src/github.com/hashicorp/hcl/hcl/parser/test-fixtures/comment_single.hcl deleted file mode 100644 index fec5601..0000000 --- a/newt/Godeps/_workspace/src/github.com/hashicorp/hcl/hcl/parser/test-fixtures/comment_single.hcl +++ /dev/null @@ -1 +0,0 @@ -# Hello http://git-wip-us.apache.org/repos/asf/incubator-mynewt-newt/blob/4ae9684f/newt/Godeps/_workspace/src/github.com/hashicorp/hcl/hcl/parser/test-fixtures/complex.hcl ---------------------------------------------------------------------- diff --git a/newt/Godeps/_workspace/src/github.com/hashicorp/hcl/hcl/parser/test-fixtures/complex.hcl b/newt/Godeps/_workspace/src/github.com/hashicorp/hcl/hcl/parser/test-fixtures/complex.hcl deleted file mode 100644 index 13b3c27..0000000 --- a/newt/Godeps/_workspace/src/github.com/hashicorp/hcl/hcl/parser/test-fixtures/complex.hcl +++ /dev/null @@ -1,42 +0,0 @@ -variable "foo" { - default = "bar" - description = "bar" -} - -variable "groups" { } - -provider "aws" { - access_key = "foo" - secret_key = "bar" -} - -provider "do" { - api_key = "${var.foo}" -} - -resource "aws_security_group" "firewall" { - count = 5 -} - -resource aws_instance "web" { - ami = "${var.foo}" - security_groups = [ - "foo", - "${aws_security_group.firewall.foo}", - "${element(split(\",\", var.groups)}", - ] - network_interface = { - device_index = 0 - description = "Main network interface" - } -} - -resource "aws_instance" "db" { - security_groups = "${aws_security_group.firewall.*.id}" - VPC = "foo" - depends_on = ["aws_instance.web"] -} - -output "web_ip" { - value = "${aws_instance.web.private_ip}" -} http://git-wip-us.apache.org/repos/asf/incubator-mynewt-newt/blob/4ae9684f/newt/Godeps/_workspace/src/github.com/hashicorp/hcl/hcl/parser/test-fixtures/complex_key.hcl ---------------------------------------------------------------------- diff --git a/newt/Godeps/_workspace/src/github.com/hashicorp/hcl/hcl/parser/test-fixtures/complex_key.hcl b/newt/Godeps/_workspace/src/github.com/hashicorp/hcl/hcl/parser/test-fixtures/complex_key.hcl deleted file mode 100644 index 0007aaf..0000000 --- a/newt/Godeps/_workspace/src/github.com/hashicorp/hcl/hcl/parser/test-fixtures/complex_key.hcl +++ /dev/null @@ -1 +0,0 @@ -foo.bar = "baz" http://git-wip-us.apache.org/repos/asf/incubator-mynewt-newt/blob/4ae9684f/newt/Godeps/_workspace/src/github.com/hashicorp/hcl/hcl/parser/test-fixtures/empty.hcl ---------------------------------------------------------------------- diff --git a/newt/Godeps/_workspace/src/github.com/hashicorp/hcl/hcl/parser/test-fixtures/empty.hcl b/newt/Godeps/_workspace/src/github.com/hashicorp/hcl/hcl/parser/test-fixtures/empty.hcl deleted file mode 100644 index e69de29..0000000 http://git-wip-us.apache.org/repos/asf/incubator-mynewt-newt/blob/4ae9684f/newt/Godeps/_workspace/src/github.com/hashicorp/hcl/hcl/parser/test-fixtures/list.hcl ---------------------------------------------------------------------- diff --git a/newt/Godeps/_workspace/src/github.com/hashicorp/hcl/hcl/parser/test-fixtures/list.hcl b/newt/Godeps/_workspace/src/github.com/hashicorp/hcl/hcl/parser/test-fixtures/list.hcl deleted file mode 100644 index 059d4ce..0000000 --- a/newt/Godeps/_workspace/src/github.com/hashicorp/hcl/hcl/parser/test-fixtures/list.hcl +++ /dev/null @@ -1 +0,0 @@ -foo = [1, 2, "foo"] http://git-wip-us.apache.org/repos/asf/incubator-mynewt-newt/blob/4ae9684f/newt/Godeps/_workspace/src/github.com/hashicorp/hcl/hcl/parser/test-fixtures/list_comma.hcl ---------------------------------------------------------------------- diff --git a/newt/Godeps/_workspace/src/github.com/hashicorp/hcl/hcl/parser/test-fixtures/list_comma.hcl b/newt/Godeps/_workspace/src/github.com/hashicorp/hcl/hcl/parser/test-fixtures/list_comma.hcl deleted file mode 100644 index 50f4218..0000000 --- a/newt/Godeps/_workspace/src/github.com/hashicorp/hcl/hcl/parser/test-fixtures/list_comma.hcl +++ /dev/null @@ -1 +0,0 @@ -foo = [1, 2, "foo",] http://git-wip-us.apache.org/repos/asf/incubator-mynewt-newt/blob/4ae9684f/newt/Godeps/_workspace/src/github.com/hashicorp/hcl/hcl/parser/test-fixtures/missing_braces.hcl ---------------------------------------------------------------------- diff --git a/newt/Godeps/_workspace/src/github.com/hashicorp/hcl/hcl/parser/test-fixtures/missing_braces.hcl b/newt/Godeps/_workspace/src/github.com/hashicorp/hcl/hcl/parser/test-fixtures/missing_braces.hcl deleted file mode 100644 index 68e7274..0000000 --- a/newt/Godeps/_workspace/src/github.com/hashicorp/hcl/hcl/parser/test-fixtures/missing_braces.hcl +++ /dev/null @@ -1,4 +0,0 @@ -# should error, but not crash -resource "template_file" "cloud_config" { - template = "$file("${path.module}/some/path")" -} http://git-wip-us.apache.org/repos/asf/incubator-mynewt-newt/blob/4ae9684f/newt/Godeps/_workspace/src/github.com/hashicorp/hcl/hcl/parser/test-fixtures/multiple.hcl ---------------------------------------------------------------------- diff --git a/newt/Godeps/_workspace/src/github.com/hashicorp/hcl/hcl/parser/test-fixtures/multiple.hcl b/newt/Godeps/_workspace/src/github.com/hashicorp/hcl/hcl/parser/test-fixtures/multiple.hcl deleted file mode 100644 index 029c54b..0000000 --- a/newt/Godeps/_workspace/src/github.com/hashicorp/hcl/hcl/parser/test-fixtures/multiple.hcl +++ /dev/null @@ -1,2 +0,0 @@ -foo = "bar" -key = 7 http://git-wip-us.apache.org/repos/asf/incubator-mynewt-newt/blob/4ae9684f/newt/Godeps/_workspace/src/github.com/hashicorp/hcl/hcl/parser/test-fixtures/old.hcl ---------------------------------------------------------------------- diff --git a/newt/Godeps/_workspace/src/github.com/hashicorp/hcl/hcl/parser/test-fixtures/old.hcl b/newt/Godeps/_workspace/src/github.com/hashicorp/hcl/hcl/parser/test-fixtures/old.hcl deleted file mode 100644 index e9f77ca..0000000 --- a/newt/Godeps/_workspace/src/github.com/hashicorp/hcl/hcl/parser/test-fixtures/old.hcl +++ /dev/null @@ -1,3 +0,0 @@ -default = { - "eu-west-1": "ami-b1cf19c6", -} http://git-wip-us.apache.org/repos/asf/incubator-mynewt-newt/blob/4ae9684f/newt/Godeps/_workspace/src/github.com/hashicorp/hcl/hcl/parser/test-fixtures/structure.hcl ---------------------------------------------------------------------- diff --git a/newt/Godeps/_workspace/src/github.com/hashicorp/hcl/hcl/parser/test-fixtures/structure.hcl b/newt/Godeps/_workspace/src/github.com/hashicorp/hcl/hcl/parser/test-fixtures/structure.hcl deleted file mode 100644 index 92592fb..0000000 --- a/newt/Godeps/_workspace/src/github.com/hashicorp/hcl/hcl/parser/test-fixtures/structure.hcl +++ /dev/null @@ -1,5 +0,0 @@ -// This is a test structure for the lexer -foo bar "baz" { - key = 7 - foo = "bar" -} http://git-wip-us.apache.org/repos/asf/incubator-mynewt-newt/blob/4ae9684f/newt/Godeps/_workspace/src/github.com/hashicorp/hcl/hcl/parser/test-fixtures/structure_basic.hcl ---------------------------------------------------------------------- diff --git a/newt/Godeps/_workspace/src/github.com/hashicorp/hcl/hcl/parser/test-fixtures/structure_basic.hcl b/newt/Godeps/_workspace/src/github.com/hashicorp/hcl/hcl/parser/test-fixtures/structure_basic.hcl deleted file mode 100644 index 7229a1f..0000000 --- a/newt/Godeps/_workspace/src/github.com/hashicorp/hcl/hcl/parser/test-fixtures/structure_basic.hcl +++ /dev/null @@ -1,5 +0,0 @@ -foo { - value = 7 - "value" = 8 - "complex::value" = 9 -} http://git-wip-us.apache.org/repos/asf/incubator-mynewt-newt/blob/4ae9684f/newt/Godeps/_workspace/src/github.com/hashicorp/hcl/hcl/parser/test-fixtures/structure_empty.hcl ---------------------------------------------------------------------- diff --git a/newt/Godeps/_workspace/src/github.com/hashicorp/hcl/hcl/parser/test-fixtures/structure_empty.hcl b/newt/Godeps/_workspace/src/github.com/hashicorp/hcl/hcl/parser/test-fixtures/structure_empty.hcl deleted file mode 100644 index 4d156dd..0000000 --- a/newt/Godeps/_workspace/src/github.com/hashicorp/hcl/hcl/parser/test-fixtures/structure_empty.hcl +++ /dev/null @@ -1 +0,0 @@ -resource "foo" "bar" {} http://git-wip-us.apache.org/repos/asf/incubator-mynewt-newt/blob/4ae9684f/newt/Godeps/_workspace/src/github.com/hashicorp/hcl/hcl/parser/test-fixtures/types.hcl ---------------------------------------------------------------------- diff --git a/newt/Godeps/_workspace/src/github.com/hashicorp/hcl/hcl/parser/test-fixtures/types.hcl b/newt/Godeps/_workspace/src/github.com/hashicorp/hcl/hcl/parser/test-fixtures/types.hcl deleted file mode 100644 index cf2747e..0000000 --- a/newt/Godeps/_workspace/src/github.com/hashicorp/hcl/hcl/parser/test-fixtures/types.hcl +++ /dev/null @@ -1,7 +0,0 @@ -foo = "bar" -bar = 7 -baz = [1,2,3] -foo = -12 -bar = 3.14159 -foo = true -bar = false http://git-wip-us.apache.org/repos/asf/incubator-mynewt-newt/blob/4ae9684f/newt/Godeps/_workspace/src/github.com/hashicorp/hcl/hcl/parser/test-fixtures/unterminated_object.hcl ---------------------------------------------------------------------- diff --git a/newt/Godeps/_workspace/src/github.com/hashicorp/hcl/hcl/parser/test-fixtures/unterminated_object.hcl b/newt/Godeps/_workspace/src/github.com/hashicorp/hcl/hcl/parser/test-fixtures/unterminated_object.hcl deleted file mode 100644 index 31b37c4..0000000 --- a/newt/Godeps/_workspace/src/github.com/hashicorp/hcl/hcl/parser/test-fixtures/unterminated_object.hcl +++ /dev/null @@ -1,2 +0,0 @@ -foo "baz" { - bar = "baz" http://git-wip-us.apache.org/repos/asf/incubator-mynewt-newt/blob/4ae9684f/newt/Godeps/_workspace/src/github.com/hashicorp/hcl/hcl/parser/test-fixtures/unterminated_object_2.hcl ---------------------------------------------------------------------- diff --git a/newt/Godeps/_workspace/src/github.com/hashicorp/hcl/hcl/parser/test-fixtures/unterminated_object_2.hcl b/newt/Godeps/_workspace/src/github.com/hashicorp/hcl/hcl/parser/test-fixtures/unterminated_object_2.hcl deleted file mode 100644 index 294e36d..0000000 --- a/newt/Godeps/_workspace/src/github.com/hashicorp/hcl/hcl/parser/test-fixtures/unterminated_object_2.hcl +++ /dev/null @@ -1,6 +0,0 @@ -resource "aws_eip" "EIP1" { a { a { a { a { a { - count = "1" - -resource "aws_eip" "EIP2" { - count = "1" -} http://git-wip-us.apache.org/repos/asf/incubator-mynewt-newt/blob/4ae9684f/newt/Godeps/_workspace/src/github.com/hashicorp/hcl/hcl/printer/nodes.go ---------------------------------------------------------------------- diff --git a/newt/Godeps/_workspace/src/github.com/hashicorp/hcl/hcl/printer/nodes.go b/newt/Godeps/_workspace/src/github.com/hashicorp/hcl/hcl/printer/nodes.go deleted file mode 100644 index a98495c..0000000 --- a/newt/Godeps/_workspace/src/github.com/hashicorp/hcl/hcl/printer/nodes.go +++ /dev/null @@ -1,575 +0,0 @@ -package printer - -import ( - "bytes" - "fmt" - "sort" - - "github.com/hashicorp/hcl/hcl/ast" - "github.com/hashicorp/hcl/hcl/token" -) - -const ( - blank = byte(' ') - newline = byte('\n') - tab = byte('\t') - infinity = 1 << 30 // offset or line -) - -var ( - unindent = []byte("\uE123") // in the private use space -) - -type printer struct { - cfg Config - prev token.Pos - - comments []*ast.CommentGroup // may be nil, contains all comments - standaloneComments []*ast.CommentGroup // contains all standalone comments (not assigned to any node) - - enableTrace bool - indentTrace int -} - -type ByPosition []*ast.CommentGroup - -func (b ByPosition) Len() int { return len(b) } -func (b ByPosition) Swap(i, j int) { b[i], b[j] = b[j], b[i] } -func (b ByPosition) Less(i, j int) bool { return b[i].Pos().Before(b[j].Pos()) } - -// collectComments comments all standalone comments which are not lead or line -// comment -func (p *printer) collectComments(node ast.Node) { - // first collect all comments. This is already stored in - // ast.File.(comments) - ast.Walk(node, func(nn ast.Node) (ast.Node, bool) { - switch t := nn.(type) { - case *ast.File: - p.comments = t.Comments - return nn, false - } - return nn, true - }) - - standaloneComments := make(map[token.Pos]*ast.CommentGroup, 0) - for _, c := range p.comments { - standaloneComments[c.Pos()] = c - } - - // next remove all lead and line comments from the overall comment map. - // This will give us comments which are standalone, comments which are not - // assigned to any kind of node. - ast.Walk(node, func(nn ast.Node) (ast.Node, bool) { - switch t := nn.(type) { - case *ast.LiteralType: - if t.LineComment != nil { - for _, comment := range t.LineComment.List { - if _, ok := standaloneComments[comment.Pos()]; ok { - delete(standaloneComments, comment.Pos()) - } - } - } - case *ast.ObjectItem: - if t.LeadComment != nil { - for _, comment := range t.LeadComment.List { - if _, ok := standaloneComments[comment.Pos()]; ok { - delete(standaloneComments, comment.Pos()) - } - } - } - - if t.LineComment != nil { - for _, comment := range t.LineComment.List { - if _, ok := standaloneComments[comment.Pos()]; ok { - delete(standaloneComments, comment.Pos()) - } - } - } - } - - return nn, true - }) - - for _, c := range standaloneComments { - p.standaloneComments = append(p.standaloneComments, c) - } - - sort.Sort(ByPosition(p.standaloneComments)) - -} - -// output prints creates b printable HCL output and returns it. -func (p *printer) output(n interface{}) []byte { - var buf bytes.Buffer - - switch t := n.(type) { - case *ast.File: - return p.output(t.Node) - case *ast.ObjectList: - var index int - var nextItem token.Pos - var commented bool - for { - // TODO(arslan): refactor below comment printing, we have the same in objectType - for _, c := range p.standaloneComments { - for _, comment := range c.List { - if index != len(t.Items) { - nextItem = t.Items[index].Pos() - } else { - nextItem = token.Pos{Offset: infinity, Line: infinity} - } - - if comment.Pos().After(p.prev) && comment.Pos().Before(nextItem) { - // if we hit the end add newlines so we can print the comment - if index == len(t.Items) { - buf.Write([]byte{newline, newline}) - } - - buf.WriteString(comment.Text) - - buf.WriteByte(newline) - if index != len(t.Items) { - buf.WriteByte(newline) - } - } - } - } - - if index == len(t.Items) { - break - } - - buf.Write(p.output(t.Items[index])) - if !commented && index != len(t.Items)-1 { - buf.Write([]byte{newline, newline}) - } - index++ - } - case *ast.ObjectKey: - buf.WriteString(t.Token.Text) - case *ast.ObjectItem: - p.prev = t.Pos() - buf.Write(p.objectItem(t)) - case *ast.LiteralType: - buf.Write(p.literalType(t)) - case *ast.ListType: - buf.Write(p.list(t)) - case *ast.ObjectType: - buf.Write(p.objectType(t)) - default: - fmt.Printf(" unknown type: %T\n", n) - } - - return buf.Bytes() -} - -func (p *printer) literalType(lit *ast.LiteralType) []byte { - result := []byte(lit.Token.Text) - if lit.Token.Type == token.HEREDOC { - // Clear the trailing newline from heredocs - if result[len(result)-1] == '\n' { - result = result[:len(result)-1] - } - - // Poison lines 2+ so that we don't indent them - result = p.heredocIndent(result) - } - - return result -} - -// objectItem returns the printable HCL form of an object item. An object type -// starts with one/multiple keys and has a value. The value might be of any -// type. -func (p *printer) objectItem(o *ast.ObjectItem) []byte { - defer un(trace(p, fmt.Sprintf("ObjectItem: %s", o.Keys[0].Token.Text))) - var buf bytes.Buffer - - if o.LeadComment != nil { - for _, comment := range o.LeadComment.List { - buf.WriteString(comment.Text) - buf.WriteByte(newline) - } - } - - for i, k := range o.Keys { - buf.WriteString(k.Token.Text) - buf.WriteByte(blank) - - // reach end of key - if o.Assign.IsValid() && i == len(o.Keys)-1 && len(o.Keys) == 1 { - buf.WriteString("=") - buf.WriteByte(blank) - } - } - - buf.Write(p.output(o.Val)) - - if o.Val.Pos().Line == o.Keys[0].Pos().Line && o.LineComment != nil { - buf.WriteByte(blank) - for _, comment := range o.LineComment.List { - buf.WriteString(comment.Text) - } - } - - return buf.Bytes() -} - -// objectType returns the printable HCL form of an object type. An object type -// begins with a brace and ends with a brace. -func (p *printer) objectType(o *ast.ObjectType) []byte { - defer un(trace(p, "ObjectType")) - var buf bytes.Buffer - buf.WriteString("{") - buf.WriteByte(newline) - - var index int - var nextItem token.Pos - var commented bool - for { - // Print stand alone comments - for _, c := range p.standaloneComments { - for _, comment := range c.List { - // if we hit the end, last item should be the brace - if index != len(o.List.Items) { - nextItem = o.List.Items[index].Pos() - } else { - nextItem = o.Rbrace - } - - if comment.Pos().After(p.prev) && comment.Pos().Before(nextItem) { - // add newline if it's between other printed nodes - if index > 0 { - commented = true - buf.WriteByte(newline) - } - - buf.Write(p.indent([]byte(comment.Text))) - buf.WriteByte(newline) - if index != len(o.List.Items) { - buf.WriteByte(newline) // do not print on the end - } - } - } - } - - if index == len(o.List.Items) { - p.prev = o.Rbrace - break - } - - // check if we have adjacent one liner items. If yes we'll going to align - // the comments. - var aligned []*ast.ObjectItem - for _, item := range o.List.Items[index:] { - // we don't group one line lists - if len(o.List.Items) == 1 { - break - } - - // one means a oneliner with out any lead comment - // two means a oneliner with lead comment - // anything else might be something else - cur := lines(string(p.objectItem(item))) - if cur > 2 { - break - } - - curPos := item.Pos() - - nextPos := token.Pos{} - if index != len(o.List.Items)-1 { - nextPos = o.List.Items[index+1].Pos() - } - - prevPos := token.Pos{} - if index != 0 { - prevPos = o.List.Items[index-1].Pos() - } - - // fmt.Println("DEBUG ----------------") - // fmt.Printf("prev = %+v prevPos: %s\n", prev, prevPos) - // fmt.Printf("cur = %+v curPos: %s\n", cur, curPos) - // fmt.Printf("next = %+v nextPos: %s\n", next, nextPos) - - if curPos.Line+1 == nextPos.Line { - aligned = append(aligned, item) - index++ - continue - } - - if curPos.Line-1 == prevPos.Line { - aligned = append(aligned, item) - index++ - - // finish if we have a new line or comment next. This happens - // if the next item is not adjacent - if curPos.Line+1 != nextPos.Line { - break - } - continue - } - - break - } - - // put newlines if the items are between other non aligned items. - // newlines are also added if there is a standalone comment already, so - // check it too - if !commented && index != len(aligned) { - buf.WriteByte(newline) - } - - if len(aligned) >= 1 { - p.prev = aligned[len(aligned)-1].Pos() - - items := p.alignedItems(aligned) - buf.Write(p.indent(items)) - } else { - p.prev = o.List.Items[index].Pos() - - buf.Write(p.indent(p.objectItem(o.List.Items[index]))) - index++ - } - - buf.WriteByte(newline) - } - - buf.WriteString("}") - return buf.Bytes() -} - -func (p *printer) alignedItems(items []*ast.ObjectItem) []byte { - var buf bytes.Buffer - - // find the longest key and value length, needed for alignment - var longestKeyLen int // longest key length - var longestValLen int // longest value length - for _, item := range items { - key := len(item.Keys[0].Token.Text) - val := len(p.output(item.Val)) - - if key > longestKeyLen { - longestKeyLen = key - } - - if val > longestValLen { - longestValLen = val - } - } - - for i, item := range items { - if item.LeadComment != nil { - for _, comment := range item.LeadComment.List { - buf.WriteString(comment.Text) - buf.WriteByte(newline) - } - } - - for i, k := range item.Keys { - keyLen := len(k.Token.Text) - buf.WriteString(k.Token.Text) - for i := 0; i < longestKeyLen-keyLen+1; i++ { - buf.WriteByte(blank) - } - - // reach end of key - if i == len(item.Keys)-1 && len(item.Keys) == 1 { - buf.WriteString("=") - buf.WriteByte(blank) - } - } - - val := p.output(item.Val) - valLen := len(val) - buf.Write(val) - - if item.Val.Pos().Line == item.Keys[0].Pos().Line && item.LineComment != nil { - for i := 0; i < longestValLen-valLen+1; i++ { - buf.WriteByte(blank) - } - - for _, comment := range item.LineComment.List { - buf.WriteString(comment.Text) - } - } - - // do not print for the last item - if i != len(items)-1 { - buf.WriteByte(newline) - } - } - - return buf.Bytes() -} - -// list returns the printable HCL form of an list type. -func (p *printer) list(l *ast.ListType) []byte { - var buf bytes.Buffer - buf.WriteString("[") - - var longestLine int - for _, item := range l.List { - // for now we assume that the list only contains literal types - if lit, ok := item.(*ast.LiteralType); ok { - lineLen := len(lit.Token.Text) - if lineLen > longestLine { - longestLine = lineLen - } - } - } - - insertSpaceBeforeItem := false - for i, item := range l.List { - if item.Pos().Line != l.Lbrack.Line { - // multiline list, add newline before we add each item - buf.WriteByte(newline) - insertSpaceBeforeItem = false - // also indent each line - val := p.output(item) - curLen := len(val) - buf.Write(p.indent(val)) - buf.WriteString(",") - - if lit, ok := item.(*ast.LiteralType); ok && lit.LineComment != nil { - // if the next item doesn't have any comments, do not align - buf.WriteByte(blank) // align one space - for i := 0; i < longestLine-curLen; i++ { - buf.WriteByte(blank) - } - - for _, comment := range lit.LineComment.List { - buf.WriteString(comment.Text) - } - } - - if i == len(l.List)-1 { - buf.WriteByte(newline) - } - } else { - if insertSpaceBeforeItem { - buf.WriteByte(blank) - insertSpaceBeforeItem = false - } - buf.Write(p.output(item)) - if i != len(l.List)-1 { - buf.WriteString(",") - insertSpaceBeforeItem = true - } - } - - } - - buf.WriteString("]") - return buf.Bytes() -} - -// indent indents the lines of the given buffer for each non-empty line -func (p *printer) indent(buf []byte) []byte { - var prefix []byte - if p.cfg.SpacesWidth != 0 { - for i := 0; i < p.cfg.SpacesWidth; i++ { - prefix = append(prefix, blank) - } - } else { - prefix = []byte{tab} - } - - var res []byte - bol := true - for _, c := range buf { - if bol && c != '\n' { - res = append(res, prefix...) - } - - res = append(res, c) - bol = c == '\n' - } - return res -} - -// unindent removes all the indentation from the tombstoned lines -func (p *printer) unindent(buf []byte) []byte { - var res []byte - for i := 0; i < len(buf); i++ { - skip := len(buf)-i <= len(unindent) - if !skip { - skip = !bytes.Equal(unindent, buf[i:i+len(unindent)]) - } - if skip { - res = append(res, buf[i]) - continue - } - - // We have a marker. we have to backtrace here and clean out - // any whitespace ahead of our tombstone up to a \n - for j := len(res) - 1; j >= 0; j-- { - if res[j] == '\n' { - break - } - - res = res[:j] - } - - // Skip the entire unindent marker - i += len(unindent) - 1 - } - - return res -} - -// heredocIndent marks all the 2nd and further lines as unindentable -func (p *printer) heredocIndent(buf []byte) []byte { - var res []byte - bol := false - for _, c := range buf { - if bol && c != '\n' { - res = append(res, unindent...) - } - res = append(res, c) - bol = c == '\n' - } - return res -} - -func lines(txt string) int { - endline := 1 - for i := 0; i < len(txt); i++ { - if txt[i] == '\n' { - endline++ - } - } - return endline -} - -// ---------------------------------------------------------------------------- -// Tracing support - -func (p *printer) printTrace(a ...interface{}) { - if !p.enableTrace { - return - } - - const dots = ". . . . . . . . . . . . . . . . . . . . . . . . . . . . . . . . " - const n = len(dots) - i := 2 * p.indentTrace - for i > n { - fmt.Print(dots) - i -= n - } - // i <= n - fmt.Print(dots[0:i]) - fmt.Println(a...) -} - -func trace(p *printer, msg string) *printer { - p.printTrace(msg, "(") - p.indentTrace++ - return p -} - -// Usage pattern: defer un(trace(p, "...")) -func un(p *printer) { - p.indentTrace-- - p.printTrace(")") -} http://git-wip-us.apache.org/repos/asf/incubator-mynewt-newt/blob/4ae9684f/newt/Godeps/_workspace/src/github.com/hashicorp/hcl/hcl/printer/printer.go ---------------------------------------------------------------------- diff --git a/newt/Godeps/_workspace/src/github.com/hashicorp/hcl/hcl/printer/printer.go b/newt/Godeps/_workspace/src/github.com/hashicorp/hcl/hcl/printer/printer.go deleted file mode 100644 index fb9df58..0000000 --- a/newt/Godeps/_workspace/src/github.com/hashicorp/hcl/hcl/printer/printer.go +++ /dev/null @@ -1,64 +0,0 @@ -// Package printer implements printing of AST nodes to HCL format. -package printer - -import ( - "bytes" - "io" - "text/tabwriter" - - "github.com/hashicorp/hcl/hcl/ast" - "github.com/hashicorp/hcl/hcl/parser" -) - -var DefaultConfig = Config{ - SpacesWidth: 2, -} - -// A Config node controls the output of Fprint. -type Config struct { - SpacesWidth int // if set, it will use spaces instead of tabs for alignment -} - -func (c *Config) Fprint(output io.Writer, node ast.Node) error { - p := &printer{ - cfg: *c, - comments: make([]*ast.CommentGroup, 0), - standaloneComments: make([]*ast.CommentGroup, 0), - // enableTrace: true, - } - - p.collectComments(node) - - if _, err := output.Write(p.unindent(p.output(node))); err != nil { - return err - } - - // flush tabwriter, if any - var err error - if tw, _ := output.(*tabwriter.Writer); tw != nil { - err = tw.Flush() - } - - return err -} - -// Fprint "pretty-prints" an HCL node to output -// It calls Config.Fprint with default settings. -func Fprint(output io.Writer, node ast.Node) error { - return DefaultConfig.Fprint(output, node) -} - -// Format formats src HCL and returns the result. -func Format(src []byte) ([]byte, error) { - node, err := parser.Parse(src) - if err != nil { - return nil, err - } - - var buf bytes.Buffer - if err := DefaultConfig.Fprint(&buf, node); err != nil { - return nil, err - } - - return buf.Bytes(), nil -} http://git-wip-us.apache.org/repos/asf/incubator-mynewt-newt/blob/4ae9684f/newt/Godeps/_workspace/src/github.com/hashicorp/hcl/hcl/printer/printer_test.go ---------------------------------------------------------------------- diff --git a/newt/Godeps/_workspace/src/github.com/hashicorp/hcl/hcl/printer/printer_test.go b/newt/Godeps/_workspace/src/github.com/hashicorp/hcl/hcl/printer/printer_test.go deleted file mode 100644 index 86aa946..0000000 --- a/newt/Godeps/_workspace/src/github.com/hashicorp/hcl/hcl/printer/printer_test.go +++ /dev/null @@ -1,143 +0,0 @@ -package printer - -import ( - "bytes" - "errors" - "flag" - "fmt" - "io/ioutil" - "path/filepath" - "testing" - - "github.com/hashicorp/hcl/hcl/parser" -) - -var update = flag.Bool("update", false, "update golden files") - -const ( - dataDir = "testdata" -) - -type entry struct { - source, golden string -} - -// Use go test -update to create/update the respective golden files. -var data = []entry{ - {"complexhcl.input", "complexhcl.golden"}, - {"list.input", "list.golden"}, - {"comment.input", "comment.golden"}, - {"comment_aligned.input", "comment_aligned.golden"}, - {"comment_standalone.input", "comment_standalone.golden"}, -} - -func TestFiles(t *testing.T) { - for _, e := range data { - source := filepath.Join(dataDir, e.source) - golden := filepath.Join(dataDir, e.golden) - check(t, source, golden) - } -} - -func check(t *testing.T, source, golden string) { - src, err := ioutil.ReadFile(source) - if err != nil { - t.Error(err) - return - } - - res, err := format(src) - if err != nil { - t.Error(err) - return - } - - // update golden files if necessary - if *update { - if err := ioutil.WriteFile(golden, res, 0644); err != nil { - t.Error(err) - } - return - } - - // get golden - gld, err := ioutil.ReadFile(golden) - if err != nil { - t.Error(err) - return - } - - // formatted source and golden must be the same - if err := diff(source, golden, res, gld); err != nil { - t.Error(err) - return - } -} - -// diff compares a and b. -func diff(aname, bname string, a, b []byte) error { - var buf bytes.Buffer // holding long error message - - // compare lengths - if len(a) != len(b) { - fmt.Fprintf(&buf, "\nlength changed: len(%s) = %d, len(%s) = %d", aname, len(a), bname, len(b)) - } - - // compare contents - line := 1 - offs := 1 - for i := 0; i < len(a) && i < len(b); i++ { - ch := a[i] - if ch != b[i] { - fmt.Fprintf(&buf, "\n%s:%d:%d: %s", aname, line, i-offs+1, lineAt(a, offs)) - fmt.Fprintf(&buf, "\n%s:%d:%d: %s", bname, line, i-offs+1, lineAt(b, offs)) - fmt.Fprintf(&buf, "\n\n") - break - } - if ch == '\n' { - line++ - offs = i + 1 - } - } - - if buf.Len() > 0 { - return errors.New(buf.String()) - } - return nil -} - -// format parses src, prints the corresponding AST, verifies the resulting -// src is syntactically correct, and returns the resulting src or an error -// if any. -func format(src []byte) ([]byte, error) { - // parse src - node, err := parser.Parse(src) - if err != nil { - return nil, fmt.Errorf("parse: %s\n%s", err, src) - } - - var buf bytes.Buffer - - cfg := &Config{} - if err := cfg.Fprint(&buf, node); err != nil { - return nil, fmt.Errorf("print: %s", err) - } - - // make sure formatted output is syntactically correct - res := buf.Bytes() - - if _, err := parser.Parse(src); err != nil { - return nil, fmt.Errorf("parse: %s\n%s", err, src) - } - - return res, nil -} - -// lineAt returns the line in text starting at offset offs. -func lineAt(text []byte, offs int) []byte { - i := offs - for i < len(text) && text[i] != '\n' { - i++ - } - return text[offs:i] -} http://git-wip-us.apache.org/repos/asf/incubator-mynewt-newt/blob/4ae9684f/newt/Godeps/_workspace/src/github.com/hashicorp/hcl/hcl/printer/testdata/comment.golden ---------------------------------------------------------------------- diff --git a/newt/Godeps/_workspace/src/github.com/hashicorp/hcl/hcl/printer/testdata/comment.golden b/newt/Godeps/_workspace/src/github.com/hashicorp/hcl/hcl/printer/testdata/comment.golden deleted file mode 100644 index e86215f..0000000 --- a/newt/Godeps/_workspace/src/github.com/hashicorp/hcl/hcl/printer/testdata/comment.golden +++ /dev/null @@ -1,36 +0,0 @@ -// A standalone comment is a comment which is not attached to any kind of node - -// This comes from Terraform, as a test -variable "foo" { - # Standalone comment should be still here - - default = "bar" - description = "bar" # yooo -} - -/* This is a multi line standalone -comment*/ - -// fatih arslan -/* This is a developer test -account and a multine comment */ -developer = ["fatih", "arslan"] // fatih arslan - -# One line here -numbers = [1, 2] // another line here - -# Another comment -variable = { - description = "bar" # another yooo - - foo { - # Nested standalone - - bar = "fatih" - } -} - -// lead comment -foo { - bar = "fatih" // line comment 2 -} // line comment 3 \ No newline at end of file http://git-wip-us.apache.org/repos/asf/incubator-mynewt-newt/blob/4ae9684f/newt/Godeps/_workspace/src/github.com/hashicorp/hcl/hcl/printer/testdata/comment.input ---------------------------------------------------------------------- diff --git a/newt/Godeps/_workspace/src/github.com/hashicorp/hcl/hcl/printer/testdata/comment.input b/newt/Godeps/_workspace/src/github.com/hashicorp/hcl/hcl/printer/testdata/comment.input deleted file mode 100644 index 57c37ac..0000000 --- a/newt/Godeps/_workspace/src/github.com/hashicorp/hcl/hcl/printer/testdata/comment.input +++ /dev/null @@ -1,37 +0,0 @@ -// A standalone comment is a comment which is not attached to any kind of node - - // This comes from Terraform, as a test -variable "foo" { - # Standalone comment should be still here - - default = "bar" - description = "bar" # yooo -} - -/* This is a multi line standalone -comment*/ - - -// fatih arslan -/* This is a developer test -account and a multine comment */ -developer = [ "fatih", "arslan"] // fatih arslan - -# One line here -numbers = [1,2] // another line here - - # Another comment -variable = { - description = "bar" # another yooo - foo { - # Nested standalone - - bar = "fatih" - } -} - - // lead comment -foo { - bar = "fatih" // line comment 2 -} // line comment 3 - http://git-wip-us.apache.org/repos/asf/incubator-mynewt-newt/blob/4ae9684f/newt/Godeps/_workspace/src/github.com/hashicorp/hcl/hcl/printer/testdata/comment_aligned.golden ---------------------------------------------------------------------- diff --git a/newt/Godeps/_workspace/src/github.com/hashicorp/hcl/hcl/printer/testdata/comment_aligned.golden b/newt/Godeps/_workspace/src/github.com/hashicorp/hcl/hcl/printer/testdata/comment_aligned.golden deleted file mode 100644 index e8469e5..0000000 --- a/newt/Godeps/_workspace/src/github.com/hashicorp/hcl/hcl/printer/testdata/comment_aligned.golden +++ /dev/null @@ -1,32 +0,0 @@ -aligned { - # We have some aligned items below - foo = "fatih" # yoo1 - default = "bar" # yoo2 - bar = "bar and foo" # yoo3 - - default = { - bar = "example" - } - - #deneme arslan - fatih = ["fatih"] # yoo4 - - #fatih arslan - fatiharslan = ["arslan"] // yoo5 - - default = { - bar = "example" - } - - security_groups = [ - "foo", # kenya 1 - "${aws_security_group.firewall.foo}", # kenya 2 - ] - - security_groups2 = [ - "foo", # kenya 1 - "bar", # kenya 1.5 - "${aws_security_group.firewall.foo}", # kenya 2 - "foobar", # kenya 3 - ] -} \ No newline at end of file http://git-wip-us.apache.org/repos/asf/incubator-mynewt-newt/blob/4ae9684f/newt/Godeps/_workspace/src/github.com/hashicorp/hcl/hcl/printer/testdata/comment_aligned.input ---------------------------------------------------------------------- diff --git a/newt/Godeps/_workspace/src/github.com/hashicorp/hcl/hcl/printer/testdata/comment_aligned.input b/newt/Godeps/_workspace/src/github.com/hashicorp/hcl/hcl/printer/testdata/comment_aligned.input deleted file mode 100644 index bd43ab1..0000000 --- a/newt/Godeps/_workspace/src/github.com/hashicorp/hcl/hcl/printer/testdata/comment_aligned.input +++ /dev/null @@ -1,28 +0,0 @@ -aligned { -# We have some aligned items below - foo = "fatih" # yoo1 - default = "bar" # yoo2 - bar = "bar and foo" # yoo3 - default = { - bar = "example" - } - #deneme arslan - fatih = ["fatih"] # yoo4 - #fatih arslan - fatiharslan = ["arslan"] // yoo5 - default = { - bar = "example" - } - -security_groups = [ - "foo", # kenya 1 - "${aws_security_group.firewall.foo}", # kenya 2 -] - -security_groups2 = [ - "foo", # kenya 1 - "bar", # kenya 1.5 - "${aws_security_group.firewall.foo}", # kenya 2 - "foobar", # kenya 3 -] -} http://git-wip-us.apache.org/repos/asf/incubator-mynewt-newt/blob/4ae9684f/newt/Godeps/_workspace/src/github.com/hashicorp/hcl/hcl/printer/testdata/comment_standalone.golden ---------------------------------------------------------------------- diff --git a/newt/Godeps/_workspace/src/github.com/hashicorp/hcl/hcl/printer/testdata/comment_standalone.golden b/newt/Godeps/_workspace/src/github.com/hashicorp/hcl/hcl/printer/testdata/comment_standalone.golden deleted file mode 100644 index 962dbf2..0000000 --- a/newt/Godeps/_workspace/src/github.com/hashicorp/hcl/hcl/printer/testdata/comment_standalone.golden +++ /dev/null @@ -1,16 +0,0 @@ -// A standalone comment - -aligned { - # Standalone 1 - - a = "bar" # yoo1 - default = "bar" # yoo2 - - # Standalone 2 -} - -# Standalone 3 - -numbers = [1, 2] // another line here - -# Standalone 4 http://git-wip-us.apache.org/repos/asf/incubator-mynewt-newt/blob/4ae9684f/newt/Godeps/_workspace/src/github.com/hashicorp/hcl/hcl/printer/testdata/comment_standalone.input ---------------------------------------------------------------------- diff --git a/newt/Godeps/_workspace/src/github.com/hashicorp/hcl/hcl/printer/testdata/comment_standalone.input b/newt/Godeps/_workspace/src/github.com/hashicorp/hcl/hcl/printer/testdata/comment_standalone.input deleted file mode 100644 index 4436cb1..0000000 --- a/newt/Godeps/_workspace/src/github.com/hashicorp/hcl/hcl/printer/testdata/comment_standalone.input +++ /dev/null @@ -1,16 +0,0 @@ -// A standalone comment - -aligned { - # Standalone 1 - - a = "bar" # yoo1 - default = "bar" # yoo2 - - # Standalone 2 -} - - # Standalone 3 - -numbers = [1,2] // another line here - - # Standalone 4 http://git-wip-us.apache.org/repos/asf/incubator-mynewt-newt/blob/4ae9684f/newt/Godeps/_workspace/src/github.com/hashicorp/hcl/hcl/printer/testdata/complexhcl.golden ---------------------------------------------------------------------- diff --git a/newt/Godeps/_workspace/src/github.com/hashicorp/hcl/hcl/printer/testdata/complexhcl.golden b/newt/Godeps/_workspace/src/github.com/hashicorp/hcl/hcl/printer/testdata/complexhcl.golden deleted file mode 100644 index b733a27..0000000 --- a/newt/Godeps/_workspace/src/github.com/hashicorp/hcl/hcl/printer/testdata/complexhcl.golden +++ /dev/null @@ -1,54 +0,0 @@ -variable "foo" { - default = "bar" - description = "bar" -} - -developer = ["fatih", "arslan"] - -provider "aws" { - access_key = "foo" - secret_key = "bar" -} - -provider "do" { - api_key = "${var.foo}" -} - -resource "aws_security_group" "firewall" { - count = 5 -} - -resource aws_instance "web" { - ami = "${var.foo}" - - security_groups = [ - "foo", - "${aws_security_group.firewall.foo}", - ] - - network_interface { - device_index = 0 - description = "Main network interface" - } - - network_interface = { - device_index = 1 - - description = <<EOF -ANOTHER NETWORK INTERFACE -EOF - } -} - -resource "aws_instance" "db" { - security_groups = "${aws_security_group.firewall.*.id}" - VPC = "foo" - - depends_on = ["aws_instance.web"] -} - -output "web_ip" { - value = <<EOF -TUBES -EOF -} \ No newline at end of file http://git-wip-us.apache.org/repos/asf/incubator-mynewt-newt/blob/4ae9684f/newt/Godeps/_workspace/src/github.com/hashicorp/hcl/hcl/printer/testdata/complexhcl.input ---------------------------------------------------------------------- diff --git a/newt/Godeps/_workspace/src/github.com/hashicorp/hcl/hcl/printer/testdata/complexhcl.input b/newt/Godeps/_workspace/src/github.com/hashicorp/hcl/hcl/printer/testdata/complexhcl.input deleted file mode 100644 index 7123418..0000000 --- a/newt/Godeps/_workspace/src/github.com/hashicorp/hcl/hcl/printer/testdata/complexhcl.input +++ /dev/null @@ -1,53 +0,0 @@ -variable "foo" { - default = "bar" - description = "bar" -} - -developer = [ "fatih", "arslan"] - -provider "aws" { - access_key ="foo" - secret_key = "bar" -} - - provider "do" { - api_key = "${var.foo}" -} - -resource "aws_security_group" "firewall" { - count = 5 - } - - resource aws_instance "web" { - ami = "${var.foo}" - security_groups = [ - "foo", - "${aws_security_group.firewall.foo}" - ] - - network_interface { - device_index = 0 - description = "Main network interface" - } - - network_interface = { - device_index = 1 - description = <<EOF -ANOTHER NETWORK INTERFACE -EOF - } - } - -resource "aws_instance" "db" { - security_groups = "${aws_security_group.firewall.*.id}" - VPC = "foo" - - depends_on = ["aws_instance.web"] -} - -output "web_ip" { - - value=<<EOF -TUBES -EOF -} http://git-wip-us.apache.org/repos/asf/incubator-mynewt-newt/blob/4ae9684f/newt/Godeps/_workspace/src/github.com/hashicorp/hcl/hcl/printer/testdata/list.golden ---------------------------------------------------------------------- diff --git a/newt/Godeps/_workspace/src/github.com/hashicorp/hcl/hcl/printer/testdata/list.golden b/newt/Godeps/_workspace/src/github.com/hashicorp/hcl/hcl/printer/testdata/list.golden deleted file mode 100644 index ec59e7c..0000000 --- a/newt/Godeps/_workspace/src/github.com/hashicorp/hcl/hcl/printer/testdata/list.golden +++ /dev/null @@ -1,27 +0,0 @@ -foo = ["fatih", "arslan"] - -foo = ["bar", "qaz"] - -foo = ["zeynep", - "arslan", -] - -foo = ["fatih", "zeynep", - "arslan", -] - -foo = [ - "vim-go", - "golang", - "hcl", -] - -foo = [] - -foo = [1, 2, 3, 4] - -foo = [ - "kenya", - "ethiopia", - "columbia", -] \ No newline at end of file http://git-wip-us.apache.org/repos/asf/incubator-mynewt-newt/blob/4ae9684f/newt/Godeps/_workspace/src/github.com/hashicorp/hcl/hcl/printer/testdata/list.input ---------------------------------------------------------------------- diff --git a/newt/Godeps/_workspace/src/github.com/hashicorp/hcl/hcl/printer/testdata/list.input b/newt/Godeps/_workspace/src/github.com/hashicorp/hcl/hcl/printer/testdata/list.input deleted file mode 100644 index dd68fdd..0000000 --- a/newt/Godeps/_workspace/src/github.com/hashicorp/hcl/hcl/printer/testdata/list.input +++ /dev/null @@ -1,21 +0,0 @@ -foo = ["fatih", "arslan" ] - -foo = [ "bar", "qaz", ] - -foo = [ "zeynep", -"arslan", ] - -foo = ["fatih", "zeynep", -"arslan", ] - -foo = [ - "vim-go", - "golang", "hcl"] - -foo = [] - -foo = [1, 2,3, 4] - -foo = [ - "kenya", "ethiopia", - "columbia"] http://git-wip-us.apache.org/repos/asf/incubator-mynewt-newt/blob/4ae9684f/newt/Godeps/_workspace/src/github.com/hashicorp/hcl/hcl/scanner/scanner.go ---------------------------------------------------------------------- diff --git a/newt/Godeps/_workspace/src/github.com/hashicorp/hcl/hcl/scanner/scanner.go b/newt/Godeps/_workspace/src/github.com/hashicorp/hcl/hcl/scanner/scanner.go deleted file mode 100644 index 87bd504..0000000 --- a/newt/Godeps/_workspace/src/github.com/hashicorp/hcl/hcl/scanner/scanner.go +++ /dev/null @@ -1,612 +0,0 @@ -// Package scanner implements a scanner for HCL (HashiCorp Configuration -// Language) source text. -package scanner - -import ( - "bytes" - "fmt" - "os" - "unicode" - "unicode/utf8" - - "github.com/hashicorp/hcl/hcl/token" -) - -// eof represents a marker rune for the end of the reader. -const eof = rune(0) - -// Scanner defines a lexical scanner -type Scanner struct { - buf *bytes.Buffer // Source buffer for advancing and scanning - src []byte // Source buffer for immutable access - - // Source Position - srcPos token.Pos // current position - prevPos token.Pos // previous position, used for peek() method - - lastCharLen int // length of last character in bytes - lastLineLen int // length of last line in characters (for correct column reporting) - - tokStart int // token text start position - tokEnd int // token text end position - - // Error is called for each error encountered. If no Error - // function is set, the error is reported to os.Stderr. - Error func(pos token.Pos, msg string) - - // ErrorCount is incremented by one for each error encountered. - ErrorCount int - - // tokPos is the start position of most recently scanned token; set by - // Scan. The Filename field is always left untouched by the Scanner. If - // an error is reported (via Error) and Position is invalid, the scanner is - // not inside a token. - tokPos token.Pos -} - -// New creates and initializes a new instance of Scanner using src as -// its source content. -func New(src []byte) *Scanner { - // even though we accept a src, we read from a io.Reader compatible type - // (*bytes.Buffer). So in the future we might easily change it to streaming - // read. - b := bytes.NewBuffer(src) - s := &Scanner{ - buf: b, - src: src, - } - - // srcPosition always starts with 1 - s.srcPos.Line = 1 - return s -} - -// next reads the next rune from the bufferred reader. Returns the rune(0) if -// an error occurs (or io.EOF is returned). -func (s *Scanner) next() rune { - ch, size, err := s.buf.ReadRune() - if err != nil { - // advance for error reporting - s.srcPos.Column++ - s.srcPos.Offset += size - s.lastCharLen = size - return eof - } - - if ch == utf8.RuneError && size == 1 { - s.srcPos.Column++ - s.srcPos.Offset += size - s.lastCharLen = size - s.err("illegal UTF-8 encoding") - return ch - } - - // remember last position - s.prevPos = s.srcPos - - s.srcPos.Column++ - s.lastCharLen = size - s.srcPos.Offset += size - - if ch == '\n' { - s.srcPos.Line++ - s.lastLineLen = s.srcPos.Column - s.srcPos.Column = 0 - } - - // debug - // fmt.Printf("ch: %q, offset:column: %d:%d\n", ch, s.srcPos.Offset, s.srcPos.Column) - return ch -} - -// unread unreads the previous read Rune and updates the source position -func (s *Scanner) unread() { - if err := s.buf.UnreadRune(); err != nil { - panic(err) // this is user fault, we should catch it - } - s.srcPos = s.prevPos // put back last position -} - -// peek returns the next rune without advancing the reader. -func (s *Scanner) peek() rune { - peek, _, err := s.buf.ReadRune() - if err != nil { - return eof - } - - s.buf.UnreadRune() - return peek -} - -// Scan scans the next token and returns the token. -func (s *Scanner) Scan() token.Token { - ch := s.next() - - // skip white space - for isWhitespace(ch) { - ch = s.next() - } - - var tok token.Type - - // token text markings - s.tokStart = s.srcPos.Offset - s.lastCharLen - - // token position, initial next() is moving the offset by one(size of rune - // actually), though we are interested with the starting point - s.tokPos.Offset = s.srcPos.Offset - s.lastCharLen - if s.srcPos.Column > 0 { - // common case: last character was not a '\n' - s.tokPos.Line = s.srcPos.Line - s.tokPos.Column = s.srcPos.Column - } else { - // last character was a '\n' - // (we cannot be at the beginning of the source - // since we have called next() at least once) - s.tokPos.Line = s.srcPos.Line - 1 - s.tokPos.Column = s.lastLineLen - } - - switch { - case isLetter(ch): - tok = token.IDENT - lit := s.scanIdentifier() - if lit == "true" || lit == "false" { - tok = token.BOOL - } - case isDecimal(ch): - tok = s.scanNumber(ch) - default: - switch ch { - case eof: - tok = token.EOF - case '"': - tok = token.STRING - s.scanString() - case '#', '/': - tok = token.COMMENT - s.scanComment(ch) - case '.': - tok = token.PERIOD - ch = s.peek() - if isDecimal(ch) { - tok = token.FLOAT - ch = s.scanMantissa(ch) - ch = s.scanExponent(ch) - } - case '<': - tok = token.HEREDOC - s.scanHeredoc() - case '[': - tok = token.LBRACK - case ']': - tok = token.RBRACK - case '{': - tok = token.LBRACE - case '}': - tok = token.RBRACE - case ',': - tok = token.COMMA - case '=': - tok = token.ASSIGN - case '+': - tok = token.ADD - case '-': - if isDecimal(s.peek()) { - ch := s.next() - tok = s.scanNumber(ch) - } else { - tok = token.SUB - } - default: - s.err("illegal char") - } - } - - // finish token ending - s.tokEnd = s.srcPos.Offset - - // create token literal - var tokenText string - if s.tokStart >= 0 { - tokenText = string(s.src[s.tokStart:s.tokEnd]) - } - s.tokStart = s.tokEnd // ensure idempotency of tokenText() call - - return token.Token{ - Type: tok, - Pos: s.tokPos, - Text: tokenText, - } -} - -func (s *Scanner) scanComment(ch rune) { - // single line comments - if ch == '#' || (ch == '/' && s.peek() != '*') { - ch = s.next() - for ch != '\n' && ch >= 0 && ch != eof { - ch = s.next() - } - if ch != eof && ch >= 0 { - s.unread() - } - return - } - - // be sure we get the character after /* This allows us to find comment's - // that are not erminated - if ch == '/' { - s.next() - ch = s.next() // read character after "/*" - } - - // look for /* - style comments - for { - if ch < 0 || ch == eof { - s.err("comment not terminated") - break - } - - ch0 := ch - ch = s.next() - if ch0 == '*' && ch == '/' { - break - } - } -} - -// scanNumber scans a HCL number definition starting with the given rune -func (s *Scanner) scanNumber(ch rune) token.Type { - if ch == '0' { - // check for hexadecimal, octal or float - ch = s.next() - if ch == 'x' || ch == 'X' { - // hexadecimal - ch = s.next() - found := false - for isHexadecimal(ch) { - ch = s.next() - found = true - } - - if !found { - s.err("illegal hexadecimal number") - } - - if ch != eof { - s.unread() - } - - return token.NUMBER - } - - // now it's either something like: 0421(octal) or 0.1231(float) - illegalOctal := false - for isDecimal(ch) { - ch = s.next() - if ch == '8' || ch == '9' { - // this is just a possibility. For example 0159 is illegal, but - // 0159.23 is valid. So we mark a possible illegal octal. If - // the next character is not a period, we'll print the error. - illegalOctal = true - } - } - - if ch == 'e' || ch == 'E' { - ch = s.scanExponent(ch) - return token.FLOAT - } - - if ch == '.' { - ch = s.scanFraction(ch) - - if ch == 'e' || ch == 'E' { - ch = s.next() - ch = s.scanExponent(ch) - } - return token.FLOAT - } - - if illegalOctal { - s.err("illegal octal number") - } - - if ch != eof { - s.unread() - } - return token.NUMBER - } - - s.scanMantissa(ch) - ch = s.next() // seek forward - if ch == 'e' || ch == 'E' { - ch = s.scanExponent(ch) - return token.FLOAT - } - - if ch == '.' { - ch = s.scanFraction(ch) - if ch == 'e' || ch == 'E' { - ch = s.next() - ch = s.scanExponent(ch) - } - return token.FLOAT - } - - if ch != eof { - s.unread() - } - return token.NUMBER -} - -// scanMantissa scans the mantissa begining from the rune. It returns the next -// non decimal rune. It's used to determine wheter it's a fraction or exponent. -func (s *Scanner) scanMantissa(ch rune) rune { - scanned := false - for isDecimal(ch) { - ch = s.next() - scanned = true - } - - if scanned && ch != eof { - s.unread() - } - return ch -} - -// scanFraction scans the fraction after the '.' rune -func (s *Scanner) scanFraction(ch rune) rune { - if ch == '.' { - ch = s.peek() // we peek just to see if we can move forward - ch = s.scanMantissa(ch) - } - return ch -} - -// scanExponent scans the remaining parts of an exponent after the 'e' or 'E' -// rune. -func (s *Scanner) scanExponent(ch rune) rune { - if ch == 'e' || ch == 'E' { - ch = s.next() - if ch == '-' || ch == '+' { - ch = s.next() - } - ch = s.scanMantissa(ch) - } - return ch -} - -// scanHeredoc scans a heredoc string. -func (s *Scanner) scanHeredoc() { - // Scan the second '<' in example: '<<EOF' - if s.next() != '<' { - s.err("heredoc expected second '<', didn't see it") - return - } - - // Get the original offset so we can read just the heredoc ident - offs := s.srcPos.Offset - - // Scan the identifier - ch := s.next() - for isLetter(ch) || isDigit(ch) { - ch = s.next() - } - - // If we reached an EOF then that is not good - if ch == eof { - s.err("heredoc not terminated") - return - } - - // Ignore the '\r' in Windows line endings - if ch == '\r' { - if s.peek() == '\n' { - ch = s.next() - } - } - - // If we didn't reach a newline then that is also not good - if ch != '\n' { - s.err("invalid characters in heredoc anchor") - return - } - - // Read the identifier - identBytes := s.src[offs : s.srcPos.Offset-s.lastCharLen] - - // Read the actual string value - lineStart := s.srcPos.Offset - for { - ch := s.next() - - // Special newline handling. - if ch == '\n' { - // Math is fast, so we first compare the byte counts to - // see if we have a chance of seeing the same identifier. If those - // match, then we compare the string values directly. - lineBytesLen := s.srcPos.Offset - s.lastCharLen - lineStart - if lineBytesLen == len(identBytes) && - bytes.Equal(identBytes, s.src[lineStart:s.srcPos.Offset-s.lastCharLen]) { - break - } - - // Not an anchor match, record the start of a new line - lineStart = s.srcPos.Offset - } - - if ch == eof { - s.err("heredoc not terminated") - return - } - } - - return -} - -// scanString scans a quoted string -func (s *Scanner) scanString() { - braces := 0 - for { - // '"' opening already consumed - // read character after quote - ch := s.next() - - if ch == '\n' || ch < 0 || ch == eof { - s.err("literal not terminated") - return - } - - if ch == '"' && braces == 0 { - break - } - - // If we're going into a ${} then we can ignore quotes for awhile - if braces == 0 && ch == '$' && s.peek() == '{' { - braces++ - s.next() - } else if braces > 0 && ch == '{' { - braces++ - } - if braces > 0 && ch == '}' { - braces-- - } - - if ch == '\\' { - s.scanEscape() - } - } - - return -} - -// scanEscape scans an escape sequence -func (s *Scanner) scanEscape() rune { - // http://en.cppreference.com/w/cpp/language/escape - ch := s.next() // read character after '/' - switch ch { - case 'a', 'b', 'f', 'n', 'r', 't', 'v', '\\', '"': - // nothing to do - case '0', '1', '2', '3', '4', '5', '6', '7': - // octal notation - ch = s.scanDigits(ch, 8, 3) - case 'x': - // hexademical notation - ch = s.scanDigits(s.next(), 16, 2) - case 'u': - // universal character name - ch = s.scanDigits(s.next(), 16, 4) - case 'U': - // universal character name - ch = s.scanDigits(s.next(), 16, 8) - default: - s.err("illegal char escape") - } - return ch -} - -// scanDigits scans a rune with the given base for n times. For example an -// octal notation \184 would yield in scanDigits(ch, 8, 3) -func (s *Scanner) scanDigits(ch rune, base, n int) rune { - for n > 0 && digitVal(ch) < base { - ch = s.next() - n-- - } - if n > 0 { - s.err("illegal char escape") - } - - // we scanned all digits, put the last non digit char back - s.unread() - return ch -} - -// scanIdentifier scans an identifier and returns the literal string -func (s *Scanner) scanIdentifier() string { - offs := s.srcPos.Offset - s.lastCharLen - ch := s.next() - for isLetter(ch) || isDigit(ch) || ch == '-' || ch == '.' { - ch = s.next() - } - - if ch != eof { - s.unread() // we got identifier, put back latest char - } - - return string(s.src[offs:s.srcPos.Offset]) -} - -// recentPosition returns the position of the character immediately after the -// character or token returned by the last call to Scan. -func (s *Scanner) recentPosition() (pos token.Pos) { - pos.Offset = s.srcPos.Offset - s.lastCharLen - switch { - case s.srcPos.Column > 0: - // common case: last character was not a '\n' - pos.Line = s.srcPos.Line - pos.Column = s.srcPos.Column - case s.lastLineLen > 0: - // last character was a '\n' - // (we cannot be at the beginning of the source - // since we have called next() at least once) - pos.Line = s.srcPos.Line - 1 - pos.Column = s.lastLineLen - default: - // at the beginning of the source - pos.Line = 1 - pos.Column = 1 - } - return -} - -// err prints the error of any scanning to s.Error function. If the function is -// not defined, by default it prints them to os.Stderr -func (s *Scanner) err(msg string) { - s.ErrorCount++ - pos := s.recentPosition() - - if s.Error != nil { - s.Error(pos, msg) - return - } - - fmt.Fprintf(os.Stderr, "%s: %s\n", pos, msg) -} - -// isHexadecimal returns true if the given rune is a letter -func isLetter(ch rune) bool { - return 'a' <= ch && ch <= 'z' || 'A' <= ch && ch <= 'Z' || ch == '_' || ch >= 0x80 && unicode.IsLetter(ch) -} - -// isDigit returns true if the given rune is a decimal digit -func isDigit(ch rune) bool { - return '0' <= ch && ch <= '9' || ch >= 0x80 && unicode.IsDigit(ch) -} - -// isDecimal returns true if the given rune is a decimal number -func isDecimal(ch rune) bool { - return '0' <= ch && ch <= '9' -} - -// isHexadecimal returns true if the given rune is an hexadecimal number -func isHexadecimal(ch rune) bool { - return '0' <= ch && ch <= '9' || 'a' <= ch && ch <= 'f' || 'A' <= ch && ch <= 'F' -} - -// isWhitespace returns true if the rune is a space, tab, newline or carriage return -func isWhitespace(ch rune) bool { - return ch == ' ' || ch == '\t' || ch == '\n' || ch == '\r' -} - -// digitVal returns the integer value of a given octal,decimal or hexadecimal rune -func digitVal(ch rune) int { - switch { - case '0' <= ch && ch <= '9': - return int(ch - '0') - case 'a' <= ch && ch <= 'f': - return int(ch - 'a' + 10) - case 'A' <= ch && ch <= 'F': - return int(ch - 'A' + 10) - } - return 16 // larger than any legal digit val -}
