Syntax Highlighting Mostly Done

editor.css

@@ -39,19 +39,12 @@ November 2023
 .idle-light .output { color: #000000 }
 .idle-light .error { color: #000000 }
 .idle-light .literal { color: #000000 }
-.idle-light .literal.number { color: #000000 }
 .idle-light .literal.string { color: #00aa00 }
-.idle-light .literal.string .format { color: #00aa00 }
 .idle-light .escape { color: #000000 }
 .idle-light .identifier { color: #000000 }
-.idle-light .identifier.variable { color: #000000 }
-.idle-light .identifier.constant{ color: #000000 }
 .idle-light .identifier.function { color: #0000ff }
 .idle-light .identifier.class { color: #0000ff }
-.idle-light .identifier.attribute { color: #000000 }
-.idle-light .identifier.property { color: #000000 }
 .idle-light .keyword { color: #ff7700 }
-.idle-light .keyword.operator { color: #ff7700 }
 .idle-light .builtin { color: #900090 }
 .idle-light .operator { color: #000000 }
 .idle-light .punctuation { color: #000000 }
@@ -65,17 +58,11 @@ November 2023
 .idle-dark .output { color: #ffffff }
 .idle-dark .error { color: #ffffff }
 .idle-dark .literal { color: #ffffff }
-.idle-dark .literal.number { color: #ffffff }
 .idle-dark .literal.string { color: #02ff02 }
-.idle-dark .literal.string .format { color: #02ff02 }
 .idle-dark .escape { color: #ffffff }
 .idle-dark .identifier { color: #ffffff }
-.idle-dark .identifier.variable { color: #ffffff }
-.idle-dark .identifier.constant{ color: #ffffff }
 .idle-dark .identifier.function { color: #5e5eff }
 .idle-dark .identifier.class { color: #5e5eff }
-.idle-dark .identifier.attribute { color: #ffffff }
-.idle-dark .identifier.property { color: #ffffff }
 .idle-dark .keyword { color: #ff8000 }
 .idle-dark .keyword.operator { color: #ff8000 }
 .idle-dark .builtin { color: #ff00ff }
@@ -83,29 +70,3 @@ November 2023
 .idle-dark .punctuation { color: #ffffff }
 .idle-dark .whitespace { color: #ffffff }
 .idle-dark .token-sep { display: none; }
-
-.idle-test .view-code { background-color: #442240; color: #ff0000; }
-.idle-test .edit-code { caret-color: #ff0000; }
-.idle-test .token { color: #ffffff }
-.idle-test .comment { color: #c0c0c0; font-style: italic; }
-.idle-test .output { color: #808080 }
-.idle-test .error { color: #ff0080 }
-.idle-test .literal { color: #00ffff }
-.idle-test .literal.number { color: #008080 }
-.idle-test .literal.string { color: #00ff00 }
-.idle-test .literal.string .format { color: #0000ff }
-.idle-test .escape { color: #ffff80 }
-.idle-test .identifier { color: #ffff00 }
-.idle-test .identifier.variable { color: #808000 }
-.idle-test .identifier.constant{ color: #008000 }
-.idle-test .identifier.function { color: #8080ff }
-.idle-test .identifier.class { color: #80ff80 }
-.idle-test .identifier.attribute { color: #800000 }
-.idle-test .identifier.property { color: #800080 }
-.idle-test .keyword { color: #ff8000 }
-.idle-test .keyword.operator { color: #ff8080 }
-.idle-test .builtin { color: #ff00ff }
-.idle-test .operator { color: #000080 }
-.idle-test .punctuation { color: #8000ff }
-.idle-test .whitespace { background-color: #ffffff }
-.idle-test .token-sep { background-color: #ff0000; width: 5px; }

editor.js

@@ -27,7 +27,8 @@ class PythonEditor{
     textInput(e) {
         let text = this.edit_code.innerText;
         this.view_code.innerHTML = "";
-        syntax_highlight_html(text).forEach(ele => this.view_code.appendChild(ele));
+        syntax_highlight_html(text)
+            .forEach(ele => this.view_code.appendChild(ele));
     }
 
     handleKeyDown(e) {
@@ -44,7 +45,8 @@ function insertTab() {
     if (selection.rangeCount > 0) {
         const range = selection.getRangeAt(0);
         const startOffset = range.startOffset;
-        const lineStart = range.startContainer.textContent.lastIndexOf('\n', startOffset - 1) + 1;
+        const lineStart = range.startContainer.textContent
+            .lastIndexOf('\n', startOffset - 1) + 1;
         const spacesToAdd = 4 - ((startOffset - lineStart) % 4);
         const tabSpaces = ' '.repeat(spacesToAdd);
 
@@ -71,6 +73,50 @@ function syntax_highlight(text) {
         `B'`, `B"`, `Br`, `BR`,
     ];
 
+    // const operator = "/*-+.@%^&><=";
+    const operators = [
+        "+",  "-",  "*",  "**", "/",  "//",
+        "%",  "@",  "<<", ">>", "&",  "|",
+        "^",  "~",  ":=", "<",  ">",  "<=",
+        ">=", "==", "!=", "@",  "=",  "+=",
+        "-=", "*=", "/=", "/",  "/=", "%=",
+        "@=", "&=", "|=", "^=",
+        ">>=", "<<=", "**="
+    ];
+    const delimiters = "()[]{},:.;@";
+
+    const identifier_not_start =
+        `0123456789. \n\t\\\`$?"'#()[]{},:.;@+-*/%<>&|^~:=!`;
+    const identifier_end = ` \n\t\\\`$?"'#()[]{},:.;@+-*/%<>&|^~:=!`;
+
+    const keywords = [
+        "await", "else", "import", "pass", "break", "except", "raise", "class",
+        "finally", "return", "continue", "for", "lambda", "try", "def", "from",
+        "nonlocal", "while", "assert", "global", "with", "async", "elif", "if",
+        "yield",
+    ];
+    const keywords_soft = [
+        "case", "match", "_",
+    ];
+    const keyword_operators = [
+        "in", "is", "and", "as", "del", "not", "or",
+    ];
+    const keywords_const = [
+        "False", "None", "True", "NotImplemented", "__debug__",
+    ];
+    const builtins = [
+        "abs", "aiter", "all", "any", "anext", "ascii", "bin", "bool",
+        "breakpoint", "bytearray", "bytes", "callable", "chr", "classmethod",
+        "compile", "complex", "delattr", "dict", "dir", "divmod", "enumerate",
+        "eval", "exec", "filter", "float", "format", "frozenset", "getattr",
+        "globals", "hasattr", "hash", "help", "hex", "id", "input", "int",
+        "isinstance", "issubclass", "iter", "len", "list", "locals", "map",
+        "max", "memoryview", "min", "next", "object", "oct", "open", "ord",
+        "pow", "print", "property", "range", "repr", "reversed", "round", "set",
+        "setattr", "slice", "sorted", "staticmethod", "str", "sum", "super",
+        "tuple", "type", "vars", "zip", "__import__",
+    ];
+
     const number_start = "0123456789.";
     const number_second = "0123456789bBeEjJ0OxX._";
     const number_dec_start = "123456789.";
@@ -84,6 +130,7 @@ function syntax_highlight(text) {
     const number_exp_start = "eE";
     const number_exp_second = "+-0123456789";
     const number_exp_continue = "0123456789_";
+    const number_after_radix_point = "0123456789eEjJ";
 
     const tokens = [];
     var char = text.substr(0, 1); text = text.substr(1);
@@ -110,7 +157,8 @@ function syntax_highlight(text) {
         }
 
         // String literals
-        else if (string_start.includes(char) || (string_alt_start.includes(char + text.substr(0, 1)))) {
+        else if (string_start.includes(char) ||
+                string_alt_start.includes(char + text.substr(0, 1))) {
             var depth = 1;
             var start;
             if (string_start.includes(char)) {
@@ -132,19 +180,76 @@ function syntax_highlight(text) {
                 token.token_type += "string";
                 start = char;
             }
-            if (text.substr(0, 1) == start && text.substr(1, 1) == start) depth = 3;
+            if (text.substr(0, 1) == start && text.substr(1, 1) == start)
+                depth = 3;
             token.value += char;
             char = text.substr(0, 1); text = text.substr(1);
 
             while (text.length && depth && (depth > 1 || char != '\n')) {
                 token.value += char;
-                if (char == start && depth == 1) depth--;
+                if (char == '\\') {
-                else if (char == start && text.substr(0, 1) == start && depth == 2) depth--;
+                    char = text.substr(0, 1); text = text.substr(1);
-                else if (char == start && text.substr(0, 1) == start && text.substr(1, 1) == start && depth == 3) depth--;
+                    token.value += char;
+                    char = text.substr(0, 1); text = text.substr(1);
+                    continue;
+                } else if (char == start && depth == 1) depth--;
+                else if (char == start && text.substr(0, 1) == start &&
+                        depth == 2)
+                    depth--;
+                else if (char == start && text.substr(0, 1) == start &&
+                        text.substr(1, 1) == start && depth == 3)
+                    depth--;
                 char = text.substr(0, 1); text = text.substr(1);
             }
         }
 
+        // Delimiters
+        else if ((delimiters.includes(char) ||
+                "->" == char + text.substr(0, 1)) && !(char == "." &&
+                number_after_radix_point.includes(text.substr(0, 1)))) {
+            token.token_type = "punctuation";
+            token.value += char;
+            char = text.substr(0, 1); text = text.substr(1);
+            if (">" == char) {
+                token.value += char;
+                char = text.substr(0, 1); text = text.substr(1);
+            }
+        }
+
+        // Operators
+        else if (operators.includes(char + text.substr(0, 2))) {
+            token.token_type = "operator";
+            token.value += char + text.substr(0, 2);
+            char = text.substr(2, 1); text = text.substr(3);
+        } else if (operators.includes(char + text.substr(0, 1))) {
+            token.token_type = "operator";
+            token.value += char + text.substr(0, 1);
+            char = text.substr(1, 1); text = text.substr(2);
+        } else if (operators.includes(char)) {
+            token.token_type = "operator";
+            token.value += char;
+            char = text.substr(0, 1); text = text.substr(1);
+        }
+
+        // Keywords and Identifiers
+        else if (!identifier_not_start.includes(char)) {
+            token.token_type = "identifier";
+            while (text.length && !identifier_end.includes(char)) {
+                token.value += char;
+                char = text.substr(0, 1); text = text.substr(1);
+            }
+            if (keywords.includes(token.value))
+                token.token_type = "keyword";
+            else if (keywords_soft.includes(token.value))
+                token.token_type = "keyword_soft";
+            else if (keyword_operators.includes(token.value))
+                token.token_type = "keyword_operator";
+            else if (keywords_const.includes(token.value))
+                token.token_type = "keywords_const";
+            else if (builtins.includes(token.value))
+                token.token_type = "builtin";
+        }
+
         // Number literals
         else if (number_start.includes(char)) {
             token.token_type = "number";
@@ -153,7 +258,8 @@ function syntax_highlight(text) {
                     token.value += char;
                     char = text.substr(0, 1); text = text.substr(1);
                 }
-                if (number_exp_start.includes(char) && number_exp_second.includes(text.substr(0, 1))) {
+                if (number_exp_start.includes(char) &&
+                        number_exp_second.includes(text.substr(0, 1))) {
                     token.value += char;
                     char = text.substr(0, 1); text = text.substr(1);
                     token.value += char;
@@ -197,7 +303,8 @@ function syntax_highlight(text) {
             } else if (char == 0 && "eE".includes(text.substr(0, 1))) {
                 token.value += char;
                 char = text.substr(0, 1); text = text.substr(1);
-                if (number_exp_start.includes(char) && number_exp_second.includes(text.substr(0, 1))) {
+                if (number_exp_start.includes(char) &&
+                        number_exp_second.includes(text.substr(0, 1))) {
                     token.value += char;
                     char = text.substr(0, 1); text = text.substr(1);
                     token.value += char;
@@ -211,7 +318,8 @@ function syntax_highlight(text) {
                     token.value += char;
                     char = text.substr(0, 1); text = text.substr(1);
                 }
-            } else if (char == 0 && !number_second.includes(text.substr(0, 1))) {
+            } else if (char == 0 &&
+                    !number_second.includes(text.substr(0, 1))) {
                 token.value += char;
                 char = text.substr(0, 1); text = text.substr(1);
             }
@@ -227,6 +335,65 @@ function syntax_highlight(text) {
         tokens.push({'token_type': 'sep', 'value':'·'});
     }
 
+    var last = {'token_type': 'token', 'value':''};
+    var last_not_whitespace = {'token_type': 'token', 'value':''};
+    for (var i = 0; i < tokens.length; i++) {
+        var next = {'token_type': 'token', 'value':''};
+        for (var j = i + 1; j < tokens.length; j++) {
+            if (tokens[j].token_type != "sep" &&
+                    tokens[j].token_type != "whitespace") {
+                next = tokens[j];
+                break;
+            }
+        }
+
+        if (tokens[i].token_type == "builtin" &&
+                last_not_whitespace.value == '.')
+            tokens[i].token_type = "identifier";
+        else if (tokens[i].token_type == "identifier" &&
+                last_not_whitespace.value == 'def')
+            tokens[i].token_type = "function";
+        else if (tokens[i].token_type == "identifier" &&
+                last_not_whitespace.value == 'class')
+            tokens[i].token_type = "class";
+        else if (tokens[i].token_type == "keyword_soft") {
+
+            if (tokens[i].value == "match") {
+                if (last.value.includes('\n') &&
+                        (next.token_type == "identifier" ||
+                        next.token_type == "punctuation"))
+                    tokens[i].token_type = "keyword";
+                else
+                    tokens[i].token_type = "identifier";
+            }
+
+            else if (tokens[i].value == "case") {
+                if (last.value.includes('\n') &&
+                        (next.token_type == "identifier" ||
+                        next.token_type == "punctuation" ||
+                        next.token_type == "keyword_soft"))
+                    tokens[i].token_type = "keyword";
+                else
+                    tokens[i].token_type = "identifier";
+            }
+
+            else if (tokens[i].value == "_") {
+                if (last_not_whitespace.value == "case" &&
+                        last_not_whitespace.token_type == "keyword")
+                    tokens[i].token_type = "keyword";
+                else
+                    tokens[i].token_type = "identifier";
+            }
+
+        }
+
+        if (tokens[i].token_type != "sep") {
+            last = tokens[i];
+            if (tokens[i].token_type != "whitespace")
+                last_not_whitespace = tokens[i];
+        }
+    }
+
     return tokens;
 }
 
@@ -237,20 +404,52 @@ function syntax_highlight_html(text) {
 
     tokens.forEach(token => {
         const element = document.createElement("span");
-        if (token.token_type == "whitespace")
+        if (token.token_type == "whitespace") {
             element.classList.add("whitespace");
-        else if (token.token_type == "comment")
+        } else if (token.token_type == "comment") {
             element.classList.add("comment");
+        }
+
         else if (token.token_type.substr(-6) == "string") {
             element.classList.add("literal");
             element.classList.add("string");
         } else if (token.token_type == "number") {
             element.classList.add("literal");
             element.classList.add("number");
-        }  else if (token.token_type == "sep")
+        }
+
+        else if (token.token_type == "punctuation") {
+            element.classList.add("punctuation");
+        } else if (token.token_type == "operator") {
+            element.classList.add("operator");
+        } else if (token.token_type == "identifier") {
+            element.classList.add("identifier");
+        } else if (token.token_type == "function") {
+            element.classList.add("identifier");
+            element.classList.add("function");
+        } else if (token.token_type == "class") {
+            element.classList.add("identifier");
+            element.classList.add("class");
+        }
+
+        else if (token.token_type == "keyword") {
+            element.classList.add("keyword");
+        } else if (token.token_type == "keyword_operator") {
+            element.classList.add("keyword");
+            element.classList.add("operator");
+        } else if (token.token_type == "keywords_const") {
+            element.classList.add("keyword");
+            element.classList.add("constant");
+        } else if (token.token_type == "builtin") {
+            element.classList.add("builtin");
+        }
+
+        else if (token.token_type == "sep") {
             element.classList.add("token-sep");
-        else
+        } else {
             element.classList.add("token");
+        }
+
         element.innerText = token.value;
         elements.push(element);
         last = token.token_type;
@@ -260,5 +459,5 @@ function syntax_highlight_html(text) {
 }
 
 document.addEventListener('DOMContentLoaded', () => {
-    document.querySelectorAll(".code").forEach(ele => new PythonEditor(ele))
+    document.querySelectorAll(".code").forEach(ele => new PythonEditor(ele));
 });

syntax_test.py

@@ -2,10 +2,11 @@
 
 # Segments written by Kyler or (and mostly at that) from python docs
 
-if 1900 < year < 2100 and 1 <= month <= 12 \
+def func():
-   and 1 <= day <= 31 and 0 <= hour < 24 \
+    if 1900 < year < 2100 and 1 <= month <= 12 \
-   and 0 <= minute < 60 and 0 <= second < 60:   # Looks like a valid date
+    and 1 <= day <= 31 and 0 <= hour < 24 \
-        return 1
+    and 0 <= minute < 60 and 0 <= second < 60:   # Looks like a valid date
+            return 1
 
 month_names = ['Januari', 'Februari', 'Maart',      # These are the
                'April',   'Mei',      'Juni',       # Dutch names
@@ -27,6 +28,9 @@ def perm(l):
     return r
 
 def perm2(l):
+    if len(l) <= 1:
+        return [l]
+    r = []
     for i in range(len(l)):
         s = l[:i] + l[i+1:]
         p = perm(l[:i] + l[i+1:])
@@ -41,6 +45,8 @@ re.compile("[A-Za-z_]"       # letter or underscore
            "[A-Za-z0-9_]*"   # letter, digit or underscore
           )
 
+
+
 # Some examples of formatted string literals:
 
 name = "Fred"
@@ -87,6 +93,14 @@ foo.__doc__ is None
 # test
 test"""
 
+r"hello\" world"
+
+rb"hello\" world"
+
+b"test"
+
+"hello\" world"
+
 # Some examples of integer literals:
 7
 2147483647
@@ -116,3 +130,99 @@ test"""
 1e100j
 3.14e-10j
 3.14_15_93j
+
+if x < y < z: print(x); print(y); print(z)
+
+print(sys.exception())
+
+try:
+    raise TypeError
+except:
+    print(repr(sys.exception()))
+    try:
+         raise ValueError
+    except:
+        print(repr(sys.exception()))
+    print(repr(sys.exception()))
+
+try:
+    raise ExceptionGroup("eg",
+        [ValueError(1), TypeError(2), OSError(3), OSError(4)])
+except* TypeError as e:
+    print(f'caught {type(e)} with nested {e.exceptions}')
+except* OSError as e:
+    print(f'caught {type(e)} with nested {e.exceptions}')
+
+try:
+    raise BlockingIOError
+except* BlockingIOError as e:
+    print(repr(e))
+
+# The return value of a function is determined by the last return statement executed
+def foo():
+    try:
+        return 'try'
+    finally:
+        return 'finally'
+
+with EXPRESSION as TARGET:
+    SUITE
+
+with A() as a, B() as b:
+    SUITE
+
+with A() as a:
+    with B() as b:
+        SUITE
+
+with (
+    A() as a,
+    B() as b,
+):
+    SUITE
+
+match = 1
+case = 1
+flag = False
+match (100, 200):
+    case (100, 300):  # Mismatch: 200 != 300
+        print('Case 1')
+    case (100, 200) if flag:  # Successful match, but guard fails
+        print('Case 2')
+    case (100, y):  # Matches and binds y to 200
+        print(f'Case 3, y: {y}')
+    case _:  # Pattern not attempted
+        print('Case 4, I match anything!')
+
+@f1(arg)
+@f2
+def func(): pass
+
+def whats_on_the_telly(penguin=None):
+    if penguin is None:
+        penguin = []
+    penguin.append("property of the zoo")
+    return penguin
+
+async def func(param1, param2):
+    do_stuff()
+    await some_coroutine()
+
+    async for TARGET in ITER:
+        SUITE
+    else:
+        SUITE2
+
+    async with EXPRESSION as TARGET:
+        SUITE
+
+class Foo:
+    pass
+
+class Foo(object):
+    pass
+
+@f1(arg)
+@f2
+class Foo: pass
+

test.css

@@ -0,0 +1,31 @@
+/*
+Kyler Olsen
+December 2023
+*/
+
+.ytd-test .view-code { background-color: #442240; color: #ff0000; }
+.ytd-test .edit-code { caret-color: #ff0000; }
+.ytd-test .token { color: #ffffff }
+.ytd-test .comment { color: #c0c0c0; font-style: italic; }
+.ytd-test .output { color: #808080 }
+.ytd-test .error { color: #ff0080 }
+.ytd-test .literal { color: #00ffff }
+.ytd-test .literal.number { color: #008080 }
+.ytd-test .literal.string { color: #00ff00 }
+.ytd-test .literal.string .format { color: #0000ff }
+.ytd-test .escape { color: #ffff80 }
+.ytd-test .identifier { color: #ffff00 }
+.ytd-test .identifier.variable { color: #808000 }
+.ytd-test .identifier.constant { color: #008000 }
+.ytd-test .identifier.function { color: #8080ff }
+.ytd-test .identifier.class { color: #80ff80 }
+.ytd-test .identifier.attribute { color: #800000 }
+.ytd-test .identifier.property { color: #800080 }
+.ytd-test .keyword { color: #ff8000 }
+.ytd-test .keyword.operator { color: #ff8080 }
+.ytd-test .keyword.constant { color: #ff8000; font-style: italic; }
+.ytd-test .builtin { color: #ff00ff }
+.ytd-test .operator { color: #000080 }
+.ytd-test .punctuation { color: #8000ff }
+.ytd-test .whitespace { background-color: #ffffff }
+.ytd-test .token-sep { background-color: #ff0000; width: 5px; }

test.html

@@ -22,12 +22,13 @@ November 2023
         .code {
             flex-grow: 1;
         }
-        .code.idle-test {
+        /* .code.idle-test {
             flex-grow: 2;
-        }
+        } */
     </style>
 
     <link rel="stylesheet" href="editor.css">
+    <link rel="stylesheet" href="test.css">
 
     <script src="editor.js" type="module"></script>
 </head>
@@ -36,7 +37,7 @@ November 2023
 
     <div class="flex-area">
         <div class="code idle-dark"></div>
-        <div class="code idle-test"></div>
+        <div class="code ytd-test"></div>
         <div class="code idle-light"></div>
     </div>