Feature: NeoLang supports quoted string

2017-08-02 23:28:52 +08:00
parent 1698d78439
commit 62ca97ae5b
16 changed files with 262 additions and 71 deletions
@@ -0,0 +1,9 @@
+color-scheme: {
+    name: "NeoTerm Default Color Scheme"
+    version: 1.1
+
+    colors: {
+        background: #000000
+        foreground: #212121
+    }
+}
@@ -1,4 +0,0 @@
-NeoLang {
-    version : 1
-    x-letter : ${NeoLang@version} >= 2
-}
@@ -6,11 +6,11 @@ package io.neolang.ast

 enum class NeoLangTokenType {
    NUMBER,
+    ID,
    STRING,
    BRACKET_START,
    BRACKET_END,
    COLON,
-    QUOTE,
    EOL,
    EOF,
 }
@@ -3,21 +3,26 @@ package io.neolang.ast
 /**
 * @author kiva
 */
-enum class NeoLangTokenValue(val value: String) {
-    COLON(":"),
-    BRACKET_START("{"),
-    BRACKET_END("}"),
-    QUOTE("\""),
-    EOF("");
+class NeoLangTokenValue(val value: String) {
+
+    override fun toString(): String {
+        return value
+    }

    companion object {
+        val COLON = NeoLangTokenValue(":")
+        val BRACKET_START = NeoLangTokenValue("{")
+        val BRACKET_END = NeoLangTokenValue("}")
+        val QUOTE = NeoLangTokenValue("\"")
+        val EOF = NeoLangTokenValue("<EOF>")
+
        fun wrap(tokenText: String): NeoLangTokenValue {
            return when (tokenText) {
                COLON.value -> COLON
                BRACKET_START.value -> BRACKET_START
                BRACKET_END.value -> BRACKET_END
                QUOTE.value -> QUOTE
-                else -> EOF
+                else -> NeoLangTokenValue(tokenText)
            }
        }
    }
@@ -1,12 +1,12 @@
 package io.neolang.ast.base

-import io.neolang.ast.visitor.NeoLangAstVisitor
+import io.neolang.ast.visitor.VisitorFactory

 /**
 * @author kiva
 */
 open class NeoLangAst {
-    fun visit(): NeoLangAstVisitor {
-        return NeoLangAstVisitor(this)
+    fun visit(): VisitorFactory {
+        return VisitorFactory(this)
    }
 }
@@ -0,0 +1,12 @@
+package io.neolang.ast.visitor
+
+import io.neolang.ast.base.NeoLangAst
+
+/**
+ * @author kiva
+ */
+class AstVisitor(private val ast: NeoLangAst, private val visitorCallback: IVisitorCallback) {
+    fun start() {
+        // TODO visitor
+    }
+}
@@ -0,0 +1,7 @@
+package io.neolang.ast.visitor
+
+/**
+ * @author kiva
+ */
+interface IVisitorCallback {
+}
@@ -1,8 +0,0 @@
-package io.neolang.ast.visitor
-
-import io.neolang.ast.base.NeoLangAst
-
-/**
- * @author kiva
- */
-open class NeoLangAstVisitor(ast: NeoLangAst)
@@ -0,0 +1,18 @@
+package io.neolang.ast.visitor
+
+import io.neolang.ast.base.NeoLangAst
+
+/**
+ * @author kiva
+ */
+
+class VisitorFactory(private val ast: NeoLangAst) {
+
+    fun getVisitor(callbackInterface: Class<out IVisitorCallback>): AstVisitor? {
+        try {
+            return AstVisitor(ast, callbackInterface.newInstance())
+        } catch (e: Exception) {
+            return null
+        }
+    }
+}
@@ -0,0 +1,9 @@
+package io.neolang.main
+
+import io.neolang.ast.visitor.IVisitorCallback
+
+/**
+ * @author kiva
+ */
+class DisplayAstVisitor : IVisitorCallback {
+}
@@ -1,4 +1,4 @@
-package io.neolang.command
+package io.neolang.main

 import io.neolang.parser.NeoLangParser
 import java.io.FileInputStream
@@ -20,20 +20,21 @@ class Main {
                val programCode = readFully(it)
                parser.setInputSource(programCode)
                val ast = parser.parse()
-                println("Compile `$it' -> $ast")
+
+                val visitor = ast.visit().getVisitor(DisplayAstVisitor::class.java)
+                if (visitor != null) {
+                    println("Compile `$it' -> $ast")
+                    visitor.start()
+                }
            }
            return
        }

        private fun readFully(file: String): String {
-            try {
-                FileInputStream(file).use {
-                    val bytes = ByteArray(it.available())
-                    it.read(bytes)
-                    return String(bytes)
-                }
-            } catch (e: Exception) {
-                return ""
+            FileInputStream(file).use {
+                val bytes = ByteArray(it.available())
+                it.read(bytes)
+                return String(bytes)
            }
        }
    }
@@ -11,7 +11,7 @@ import java.util.*
 * prog: group (group)*
 * group: attribute (attribute)*
 * attribute: TEXT COLON block
- * block: NUMBER | TEXT | (BRACKET_START group BRACKET_STOP)
+ * block: NUMBER | TEXT | (BRACKET_START group BRACKET_END)
 * ]
 */

@@ -30,25 +30,31 @@ class NeoLangLexer {

    internal fun lex(): List<NeoLangToken> {
        val programCode = this.programCode ?: return listOf()
+        val tokens = ArrayList<NeoLangToken>()
        currentPosition = 0
        lineNumber = 1
-        currentChar = programCode[currentPosition]

-        val tokens = ArrayList<NeoLangToken>()
-        while (currentPosition < programCode.length) {
-            val token = nextToken
-            if (token is NeoLangEOFToken) {
-                break
+        if (programCode.isNotEmpty()) {
+            currentChar = programCode[currentPosition]
+
+            while (currentPosition < programCode.length) {
+                val token = nextToken
+                if (token is NeoLangEOFToken) {
+                    break
+                }
+                tokens.add(token)
            }
-            tokens.add(token)
        }
        return tokens
    }

-    private fun moveToNextChar(): Boolean {
+    private fun moveToNextChar(eofThrow: Boolean = false): Boolean {
        val programCode = this.programCode ?: return false
        currentPosition++
        if (currentPosition >= programCode.length) {
+            if (eofThrow) {
+                throw InvalidTokenException("Unexpected EOF near `$currentChar' in line $lineNumber")
+            }
            return false
        } else {
            currentChar = programCode[currentPosition]
@@ -68,7 +74,9 @@ class NeoLangLexer {
                    ++lineNumber
                }
                // Skip white chars
-                moveToNextChar()
+                if (!moveToNextChar()) {
+                    return NeoLangEOFToken()
+                }
            }

            if (currentPosition >= programCode.length) {
@@ -78,11 +86,11 @@ class NeoLangLexer {
            val currentToken = NeoLangTokenValue.wrap(currentChar.toString())
            val token: NeoLangToken = when (currentToken) {
                NeoLangTokenValue.COLON -> {
-                    moveToNextChar()
+                    moveToNextChar(eofThrow = true)
                    NeoLangToken(NeoLangTokenType.COLON, currentToken)
                }
                NeoLangTokenValue.BRACKET_START -> {
-                    moveToNextChar()
+                    moveToNextChar(eofThrow = true)
                    NeoLangToken(NeoLangTokenType.BRACKET_START, currentToken)
                }
                NeoLangTokenValue.BRACKET_END -> {
@@ -90,14 +98,13 @@ class NeoLangLexer {
                    NeoLangToken(NeoLangTokenType.BRACKET_END, currentToken)
                }
                NeoLangTokenValue.QUOTE -> {
-                    moveToNextChar()
-                    NeoLangToken(NeoLangTokenType.QUOTE, currentToken)
+                    NeoLangToken(NeoLangTokenType.STRING, NeoLangTokenValue.wrap(getNextTokenAsString()))
                }
                else -> {
-                    if (Character.isDigit(currentChar)) {
+                    if (currentChar.isNumber()) {
                        NeoLangToken(NeoLangTokenType.NUMBER, NeoLangTokenValue.wrap(getNextTokenAsNumber()))
-                    } else if (Character.isLetterOrDigit(currentChar)) {
-                        NeoLangToken(NeoLangTokenType.STRING, NeoLangTokenValue.wrap(getNextTokenAsString()))
+                    } else if (isIdentifier(currentChar, true)) {
+                        NeoLangToken(NeoLangTokenType.ID, NeoLangTokenValue.wrap(getNextTokenAsId()))
                    } else {
                        throw InvalidTokenException("Unexpected character: " + currentChar)
                    }
@@ -108,20 +115,118 @@ class NeoLangLexer {
            return token
        }

-    private fun getNextTokenAsNumber(): String {
-        return buildString {
-            while (Character.isDigit(currentChar)) {
-                append(currentChar)
-                if (!moveToNextChar()) {
-                    break
-                }
-            }
-        }
-    }
-
    private fun getNextTokenAsString(): String {
+        // Skip start quote
+        // and a single quote is now allowed
+        moveToNextChar(eofThrow = true)
+        val builder = StringBuilder()
+
+        var loop = true
+        while (loop && currentChar != NeoLangTokenValue.QUOTE.value[0]) {
+            // Skip escaped char
+            if (currentChar == '\\') {
+                builder.append('\\')
+                moveToNextChar(eofThrow = true)
+            }
+            builder.append(currentChar)
+            loop = moveToNextChar()
+        }
+
+        // Skip end quote
+        moveToNextChar()
+        return builder.toString()
+    }
+
+    private fun getNextTokenAsNumber(): String {
+        var numberValue: Double = (currentChar.toInt() - '0'.toInt()).toDouble()
+
+        // Four types of numbers are supported:
+        // Dec(123) Hex(0x123) Oct(017) Bin(0b11)
+
+        // Dec
+        if (numberValue > 0) {
+            numberValue = getNextDecimalNumber(numberValue)
+        } else {
+            // is 0
+            if (!moveToNextChar()) {
+                return numberValue.toString()
+            }
+
+            // Hex
+            if (currentChar == 'x' || currentChar == 'X') {
+                numberValue = getNextHexNumber(numberValue)
+            } else if (currentChar == 'b' || currentChar == 'B') {
+                numberValue = getNextBinaryNumber(numberValue)
+            } else {
+                numberValue = getNextOctalNumber(numberValue)
+            }
+        }
+
+        return numberValue.toString()
+    }
+
+    private fun getNextOctalNumber(numberValue: Double): Double {
+        var value: Double = numberValue
+        var loop = true
+        while (loop && currentChar in ('0'..'7')) {
+            value = value * 8 + currentChar.toNumber()
+            loop = moveToNextChar()
+        }
+        return value
+    }
+
+    private fun getNextBinaryNumber(numberValue: Double): Double {
+        var value = numberValue
+        var loop = moveToNextChar() // skip 'b' or 'B'
+        while (loop && currentChar in ('0'..'1')) {
+            value += value * 2 + currentChar.toNumber()
+            loop = moveToNextChar()
+        }
+        return value
+    }
+
+    private fun getNextHexNumber(numberValue: Double): Double {
+        var value = numberValue
+        var loop = moveToNextChar() // skip 'x' or 'X'
+        while (loop && (currentChar.isHexNumber())) {
+            value *= 16
+            + (currentChar.toInt().and(15))
+            + if (currentChar >= 'A') 9 else 0
+            loop = moveToNextChar()
+        }
+        return value
+    }
+
+    private fun getNextDecimalNumber(numberValue: Double): Double {
+        var floatPointMeet = false
+        var floatPart: Double = 0.0
+        var floatNumberCounter = 1
+        var value = numberValue
+
+        var loop = moveToNextChar()
+        while (loop) {
+            if (currentChar.isNumber()) {
+                if (floatPointMeet) {
+                    floatPart = floatPart * 10 + currentChar.toNumber()
+                    floatNumberCounter *= 10
+                } else {
+                    value = value * 10 + currentChar.toNumber()
+                }
+                loop = moveToNextChar()
+
+            } else if (currentChar == '.') {
+                floatPointMeet = true
+                loop = moveToNextChar()
+            } else {
+                break
+            }
+        }
+        return value + floatPart / floatNumberCounter
+    }
+
+    private fun getNextTokenAsId(): String {
        return buildString {
-            while (Character.isLetterOrDigit(currentChar)) {
+            while (isIdentifier(currentChar, false)) {
                append(currentChar)
                if (!moveToNextChar()) {
                    break
@@ -130,4 +235,26 @@ class NeoLangLexer {
        }
    }

+    private fun isIdentifier(tokenChar: Char, isFirstChar: Boolean): Boolean {
+        val isId = (tokenChar in 'a'..'z')
+                || (tokenChar in 'A'..'Z')
+                || ("_-#$".contains(tokenChar))
+        return if (isFirstChar) isId else (isId || (tokenChar in '0'..'9'))
+    }
+
+    private fun Char.toNumber(): Int {
+        return if (isNumber()) {
+            this.toInt() - '0'.toInt()
+        } else 0
+    }
+
+    private fun Char.isNumber(): Boolean {
+        return this in ('0'..'9')
+    }
+
+    private fun Char.isHexNumber(): Boolean {
+        return this.isNumber()
+                || this in ('a'..'f')
+                || this in ('A'..'F')
+    }
 }
@@ -26,8 +26,11 @@ class NeoLangParser {

    private fun updateParserStatus(tokens: List<NeoLangToken>) {
        if (tokens.isEmpty()) {
-            throw ParseException("Input tokens must be non-empty")
+            // Allow empty program
+            ast = NeoLangProgramNode.emptyNode()
+            return
        }
+
        this.tokens.clear()
        this.tokens.addAll(tokens)
        currentPosition = 0
@@ -48,7 +51,7 @@ class NeoLangParser {
            return true

        } else if (errorThrow) {
-            throw InvalidTokenException("Unexpected token type " +
+            throw InvalidTokenException("Unexpected token `${currentToken.tokenValue}' typed " +
                    "`${currentToken.tokenType}' near line ${currentToken.lineNumber}, " +
                    "expected $tokenType")
        }
@@ -98,7 +101,7 @@ class NeoLangParser {

    private fun attribute(): NeoLangAttributeNode? {
        val token = currentToken ?: throw InvalidTokenException("Unexpected token: null")
-        if (match(NeoLangTokenType.STRING)) {
+        if (match(NeoLangTokenType.ID)) {
            match(NeoLangTokenType.COLON, errorThrow = true)
            val block = block() ?: NeoLangBlockNode.emptyNode()
            return NeoLangAttributeNode(NeoLangStringNode(token), block)
@@ -113,6 +116,10 @@ class NeoLangParser {
                match(NeoLangTokenType.NUMBER, errorThrow = true)
                return NeoLangBlockNode(NeoLangNumberNode(token))
            }
+            NeoLangTokenType.ID -> {
+                match(NeoLangTokenType.ID, errorThrow = true)
+                return NeoLangBlockNode(NeoLangStringNode(token))
+            }
            NeoLangTokenType.STRING -> {
                match(NeoLangTokenType.STRING, errorThrow = true)
                return NeoLangBlockNode(NeoLangStringNode(token))
@@ -127,7 +134,7 @@ class NeoLangParser {
            }

            else -> throw InvalidTokenException("Unexpected token `$token' for block, " +
-                    "expected `${NeoLangTokenType.NUMBER}', `${NeoLangTokenType.STRING}' or `${NeoLangTokenType.BRACKET_START}'")
+                    "expected `${NeoLangTokenType.NUMBER}', `${NeoLangTokenType.ID}' or `${NeoLangTokenType.BRACKET_START}'")
        }
    }

@@ -56,5 +56,5 @@ dependencies {
    compile 'com.simplecityapps:recyclerview-fastscroll:1.0.16'
 //    compile 'com.ramotion.cardslider:card-slider:0.1.0'
 //    compile 'com.github.igalata:Bubble-Picker:v0.2.4'
-    implementation project(path: ':NeoLang')
+    compile project(path: ':NeoLang')
 }
@@ -0,0 +1,10 @@
+package io.neoterm.customize.service
+
+import io.neolang.main.Main
+
+/**
+ * @author kiva
+ */
+object NeoServiceManager {
+    val main = Main()
+}
@@ -1,5 +1,6 @@
 package io.neoterm

+import io.neolang.main.Main
 import org.junit.Test

 /**
@@ -10,10 +11,7 @@ import org.junit.Test
 class NeoLangTest {
    @Test
    fun testNeoLangParser() {
-        val parser = io.neolang.parser.NeoLangParser()
-        parser.setInputSource("app: { x: {} \n x: hello \n a: 1111 \n x: { x: 123 } }")
-        val ast = parser.parse()
-        println(ast)
+        Main.main(arrayOf("NeoLang/example/color-scheme.nl"))
    }
 }