update JS

(bzr r2555)

1 files changed, 916 insertions, 170 deletions

diff --git a/src/dom/js/jsscan.c b/src/dom/js/jsscan.c
index 995c1e660..c043799dc 100644
--- a/src/dom/js/jsscan.c
+++ b/src/dom/js/jsscan.c
@@ -1,4 +1,5 @@
 /* -*- Mode: C; tab-width: 8; indent-tabs-mode: nil; c-basic-offset: 4 -*-
+ * vim: set sw=4 ts=8 et tw=78:
  *
  * ***** BEGIN LICENSE BLOCK *****
  * Version: MPL 1.1/GPL 2.0/LGPL 2.1
@@ -66,10 +67,18 @@
 #include "jsopcode.h"
 #include "jsregexp.h"
 #include "jsscan.h"
+#include "jsscript.h"
+
+#if JS_HAS_XML_SUPPORT
+#include "jsparse.h"
+#include "jsxml.h"
+#endif
 
 #define RESERVE_JAVA_KEYWORDS
 #define RESERVE_ECMA_KEYWORDS
 
+#define MAX_KEYWORD_LENGTH      12
+
 static struct keyword {
     const char  *name;
     JSTokenType tokentype;      /* JSTokenType */
@@ -79,7 +88,7 @@ static struct keyword {
     {"break",           TOK_BREAK,              JSOP_NOP,   JSVERSION_DEFAULT},
     {"case",            TOK_CASE,               JSOP_NOP,   JSVERSION_DEFAULT},
     {"continue",        TOK_CONTINUE,           JSOP_NOP,   JSVERSION_DEFAULT},
-    {"default",         TOK_DEFAULT,            JSOP_NOP,   JSVERSION_DEFAULT},
+    {js_default_str,    TOK_DEFAULT,            JSOP_NOP,   JSVERSION_DEFAULT},
     {js_delete_str,     TOK_DELETE,             JSOP_NOP,   JSVERSION_DEFAULT},
     {"do",              TOK_DO,                 JSOP_NOP,   JSVERSION_DEFAULT},
     {"else",            TOK_ELSE,               JSOP_NOP,   JSVERSION_DEFAULT},
@@ -96,7 +105,7 @@ static struct keyword {
     {js_this_str,       TOK_PRIMARY,            JSOP_THIS,  JSVERSION_DEFAULT},
     {js_true_str,       TOK_PRIMARY,            JSOP_TRUE,  JSVERSION_DEFAULT},
     {js_typeof_str,     TOK_UNARYOP,            JSOP_TYPEOF,JSVERSION_DEFAULT},
-    {"var",             TOK_VAR,                JSOP_DEFVAR,JSVERSION_DEFAULT},
+    {js_var_str,        TOK_VAR,                JSOP_DEFVAR,JSVERSION_DEFAULT},
     {js_void_str,       TOK_UNARYOP,            JSOP_VOID,  JSVERSION_DEFAULT},
     {"while",           TOK_WHILE,              JSOP_NOP,   JSVERSION_DEFAULT},
     {"with",            TOK_WITH,               JSOP_NOP,   JSVERSION_DEFAULT},
@@ -171,10 +180,13 @@ JSBool
 js_InitScanner(JSContext *cx)
 {
     struct keyword *kw;
+    size_t length;
     JSAtom *atom;
 
     for (kw = keywords; kw->name; kw++) {
-        atom = js_Atomize(cx, kw->name, strlen(kw->name), ATOM_PINNED);
+        length = strlen(kw->name);
+        JS_ASSERT(length <= MAX_KEYWORD_LENGTH);
+        atom = js_Atomize(cx, kw->name, length, ATOM_PINNED);
         if (!atom)
             return JS_FALSE;
         ATOM_SET_KEYWORD(atom, kw);
@@ -209,6 +221,42 @@ js_NewTokenStream(JSContext *cx, const jschar *base, size_t length,
     return ts;
 }
 
+#define TBMIN   64
+
+static JSBool
+GrowTokenBuf(JSStringBuffer *sb, size_t newlength)
+{
+    JSContext *cx;
+    jschar *base;
+    ptrdiff_t offset, length;
+    size_t tbsize;
+    JSArenaPool *pool;
+
+    cx = sb->data;
+    base = sb->base;
+    offset = PTRDIFF(sb->ptr, base, jschar);
+    pool = &cx->tempPool;
+    if (!base) {
+        tbsize = TBMIN * sizeof(jschar);
+        length = TBMIN - 1;
+        JS_ARENA_ALLOCATE_CAST(base, jschar *, pool, tbsize);
+    } else {
+        length = PTRDIFF(sb->limit, base, jschar);
+        tbsize = (length + 1) * sizeof(jschar);
+        length += length + 1;
+        JS_ARENA_GROW_CAST(base, jschar *, pool, tbsize, tbsize);
+    }
+    if (!base) {
+        JS_ReportOutOfMemory(cx);
+        sb->base = STRING_BUFFER_ERROR_BASE;
+        return JS_FALSE;
+    }
+    sb->base = base;
+    sb->limit = base + length;
+    sb->ptr = base + offset;
+    return JS_TRUE;
+}
+
 JS_FRIEND_API(JSTokenStream *)
 js_NewBufferTokenStream(JSContext *cx, const jschar *base, size_t length)
 {
@@ -227,6 +275,8 @@ js_NewBufferTokenStream(JSContext *cx, const jschar *base, size_t length)
     ts->userbuf.base = (jschar *)base;
     ts->userbuf.limit = (jschar *)base + length;
     ts->userbuf.ptr = (jschar *)base;
+    ts->tokenbuf.grow = GrowTokenBuf;
+    ts->tokenbuf.data = cx;
     ts->listener = cx->runtime->sourceHandler;
     ts->listenerData = cx->runtime->sourceHandlerData;
     return ts;
@@ -272,8 +322,8 @@ js_CloseTokenStream(JSContext *cx, JSTokenStream *ts)
     return !ts->file || fclose(ts->file) == 0;
 }
 
-static int
-my_fgets(char *buf, int size, FILE *file)
+JS_FRIEND_API(int)
+js_fgets(char *buf, int size, FILE *file)
 {
     int n, i, c;
     JSBool crflag;
@@ -323,7 +373,7 @@ GetChar(JSTokenStream *ts)
 
                     /* Fill ts->userbuf so that \r and \r\n convert to \n. */
                     crflag = (ts->flags & TSF_CRFLAG) != 0;
-                    len = my_fgets(cbuf, JS_LINE_LIMIT - crflag, ts->file);
+                    len = js_fgets(cbuf, JS_LINE_LIMIT - crflag, ts->file);
                     if (len <= 0) {
                         ts->flags |= TSF_EOF;
                         return EOF;
@@ -479,6 +529,11 @@ PeekChar(JSTokenStream *ts)
     return c;
 }
 
+/*
+ * Peek n chars ahead into ts.  Return true if n chars were read, false if
+ * there weren't enough characters in the input stream.  This function cannot
+ * be used to peek into or past a newline.
+ */
 static JSBool
 PeekChars(JSTokenStream *ts, intN n, jschar *cp)
 {
@@ -489,6 +544,10 @@ PeekChars(JSTokenStream *ts, intN n, jschar *cp)
         c = GetChar(ts);
         if (c == EOF)
             break;
+        if (c == '\n') {
+            UngetChar(ts, c);
+            break;
+        }
         cp[i] = (jschar)c;
     }
     for (j = i - 1; j >= 0; j--)
@@ -515,37 +574,52 @@ MatchChar(JSTokenStream *ts, int32 expect)
     return JS_FALSE;
 }
 
-JSBool
-js_ReportCompileErrorNumber(JSContext *cx, JSTokenStream *ts,
-                            JSCodeGenerator *cg, uintN flags,
-                            const uintN errorNumber, ...)
+static JSBool
+ReportCompileErrorNumber(JSContext *cx, void *handle, uintN flags,
+                         uintN errorNumber, JSErrorReport *report,
+                         JSBool charArgs, va_list ap)
 {
-    va_list ap;
-    JSErrorReporter onError;
-    JSErrorReport report;
-    jschar *tokenptr;
     JSString *linestr = NULL;
+    JSTokenStream *ts = NULL;
+    JSCodeGenerator *cg = NULL;
+#if JS_HAS_XML_SUPPORT
+    JSParseNode *pn = NULL;
+#endif
+    JSErrorReporter onError;
+    JSTokenPos *tp;
+    JSStackFrame *fp;
+    uintN index;
     char *message;
     JSBool warning;
 
-    if ((flags & JSREPORT_STRICT) && !JS_HAS_STRICT_OPTION(cx))
-        return JS_TRUE;
-
-    memset(&report, 0, sizeof (struct JSErrorReport));
-    report.flags = flags;
-    report.errorNumber = errorNumber;
+    memset(report, 0, sizeof (struct JSErrorReport));
+    report->flags = flags;
+    report->errorNumber = errorNumber;
     message = NULL;
 
-    va_start(ap, errorNumber);
     if (!js_ExpandErrorArguments(cx, js_GetErrorMessage, NULL,
-                                 errorNumber, &message, &report, &warning,
-                                 JS_TRUE, ap)) {
+                                 errorNumber, &message, report, &warning,
+                                 charArgs, ap)) {
         return JS_FALSE;
     }
-    va_end(ap);
 
     js_AddRoot(cx, &linestr, "error line buffer");
 
+    switch (flags & JSREPORT_HANDLE) {
+      case JSREPORT_TS:
+        ts = handle;
+        break;
+      case JSREPORT_CG:
+        cg = handle;
+        break;
+#if JS_HAS_XML_SUPPORT
+      case JSREPORT_PN:
+        pn = handle;
+        ts = pn->pn_ts;
+        break;
+#endif
+    }
+
     JS_ASSERT(!ts || ts->linebuf.limit < ts->linebuf.base + JS_LINE_LIMIT);
     onError = cx->errorReporter;
     if (onError) {
@@ -554,30 +628,70 @@ js_ReportCompileErrorNumber(JSContext *cx, JSTokenStream *ts,
          * We can be called with null ts from the regexp compilation functions.
          * The code generator (jsemit.c) may pass null ts and non-null cg.
          */
-        if (ts) {
-            report.filename = ts->filename;
-            report.lineno = ts->lineno;
-            linestr = js_NewStringCopyN(cx, ts->linebuf.base,
-                                        ts->linebuf.limit - ts->linebuf.base,
-                                        0);
-            report.linebuf = linestr
-                ? JS_GetStringBytes(linestr)
-                : NULL;
-            tokenptr =
-                ts->tokens[(ts->cursor + ts->lookahead) & NTOKENS_MASK].ptr;
-            report.tokenptr = linestr
-                ? report.linebuf + (tokenptr - ts->linebuf.base)
-                : NULL;
-            report.uclinebuf = linestr
-                ? JS_GetStringChars(linestr)
-                : NULL;
-            report.uctokenptr = linestr
-                ? report.uclinebuf + (tokenptr - ts->linebuf.base)
-                : NULL;
-        } else if (cg) {
-            report.filename = cg->filename;
-            report.lineno = CG_CURRENT_LINE(cg);
-        }
+        do {
+            if (ts) {
+                report->filename = ts->filename;
+#if JS_HAS_XML_SUPPORT
+                if (pn) {
+                    report->lineno = pn->pn_pos.begin.lineno;
+                    if (report->lineno != ts->lineno)
+                        break;
+                }
+#endif
+                report->lineno = ts->lineno;
+                linestr = js_NewStringCopyN(cx, ts->linebuf.base,
+                                            PTRDIFF(ts->linebuf.limit,
+                                                    ts->linebuf.base,
+                                                    jschar),
+                                            0);
+                report->linebuf = linestr
+                                 ? JS_GetStringBytes(linestr)
+                                 : NULL;
+                tp = &ts->tokens[(ts->cursor+ts->lookahead) & NTOKENS_MASK].pos;
+#if JS_HAS_XML_SUPPORT
+                if (pn)
+                    tp = &pn->pn_pos;
+#endif
+                /*
+                 * FIXME: What should instead happen here is that we should
+                 * find error-tokens in userbuf, if !ts->file.  That will
+                 * allow us to deliver a more helpful error message, which
+                 * includes all or part of the bad string or bad token.  The
+                 * code here yields something that looks truncated.
+                 * See https://bugzilla.mozilla.org/show_bug.cgi?id=352970
+                 */ 
+                index = 0;
+                if (tp->begin.lineno == tp->end.lineno) {
+                    if (tp->begin.index < ts->linepos)
+                        break;
+
+                    index = tp->begin.index - ts->linepos;
+                }
+
+                report->tokenptr = linestr ? report->linebuf + index : NULL;
+                report->uclinebuf = linestr ? JS_GetStringChars(linestr) : NULL;
+                report->uctokenptr = linestr ? report->uclinebuf + index : NULL;
+                break;
+            }
+
+            if (cg) {
+                report->filename = cg->filename;
+                report->lineno = CG_CURRENT_LINE(cg);
+                break;
+            }
+
+            /*
+             * If we can't find out where the error was based on the current frame,
+             * see if the next frame has a script/pc combo we can use.
+             */
+            for (fp = cx->fp; fp; fp = fp->down) {
+                if (fp->script && fp->pc) {
+                    report->filename = fp->script->filename;
+                    report->lineno = js_PCToLineNumber(cx, fp->script, fp->pc);
+                    break;
+                }
+            }
+        } while (0);
 
 #if JS_HAS_ERROR_EXCEPTIONS
         /*
@@ -598,54 +712,352 @@ js_ReportCompileErrorNumber(JSContext *cx, JSTokenStream *ts,
          */
 
         /*
-         * Only try to raise an exception if there isn't one already set -
-         * otherwise the exception will describe only the last compile error,
+         * Try to raise an exception only if there isn't one already set --
+         * otherwise the exception will describe the last compile-time error,
          * which is likely spurious.
          */
-        if (!(ts && (ts->flags & TSF_ERROR)))
-            if (js_ErrorToException(cx, message, &report))
+        if (!ts || !(ts->flags & TSF_ERROR)) {
+            if (js_ErrorToException(cx, message, report))
                 onError = NULL;
+        }
 
         /*
-         * Suppress any compiletime errors that don't occur at the top level.
+         * Suppress any compile-time errors that don't occur at the top level.
          * This may still fail, as interplevel may be zero in contexts where we
          * don't really want to call the error reporter, as when js is called
          * by other code which could catch the error.
          */
-        if (cx->interpLevel != 0)
+        if (cx->interpLevel != 0 && !JSREPORT_IS_WARNING(flags))
             onError = NULL;
 #endif
-        if (cx->runtime->debugErrorHook && onError) {
+        if (onError) {
             JSDebugErrorHook hook = cx->runtime->debugErrorHook;
-            /* test local in case debugErrorHook changed on another thread */
-            if (hook && !hook(cx, message, &report,
+
+            /*
+             * If debugErrorHook is present then we give it a chance to veto
+             * sending the error on to the regular error reporter.
+             */
+            if (hook && !hook(cx, message, report,
                               cx->runtime->debugErrorHookData)) {
                 onError = NULL;
             }
         }
         if (onError)
-            (*onError)(cx, message, &report);
+            (*onError)(cx, message, report);
     }
+
     if (message)
         JS_free(cx, message);
+    if (report->ucmessage)
+        JS_free(cx, (void *)report->ucmessage);
+
+    js_RemoveRoot(cx->runtime, &linestr);
+
+    if (ts && !JSREPORT_IS_WARNING(flags)) {
+        /* Set the error flag to suppress spurious reports. */
+        ts->flags |= TSF_ERROR;
+    }
+
+    return warning;
+}
+
+JSBool
+js_ReportCompileErrorNumber(JSContext *cx, void *handle, uintN flags,
+                            uintN errorNumber, ...)
+{
+    va_list ap;
+    JSErrorReport report;
+    JSBool warning;
+
+    if ((flags & JSREPORT_STRICT) && !JS_HAS_STRICT_OPTION(cx))
+        return JS_TRUE;
+
+    va_start(ap, errorNumber);
+    warning = ReportCompileErrorNumber(cx, handle, flags, errorNumber,
+                                       &report, JS_TRUE, ap);
+    va_end(ap);
+
+    /* 
+     * We have to do this here because js_ReportCompileErrorNumberUC doesn't
+     * need to do this.
+     */
     if (report.messageArgs) {
         int i = 0;
         while (report.messageArgs[i])
             JS_free(cx, (void *)report.messageArgs[i++]);
         JS_free(cx, (void *)report.messageArgs);
     }
-    if (report.ucmessage)
-        JS_free(cx, (void *)report.ucmessage);
 
-    js_RemoveRoot(cx->runtime, &linestr);
+    return warning;
+}
+
+JSBool
+js_ReportCompileErrorNumberUC(JSContext *cx, void *handle, uintN flags,
+                              uintN errorNumber, ...)
+{
+    va_list ap;
+    JSErrorReport report;
+    JSBool warning;
+
+    if ((flags & JSREPORT_STRICT) && !JS_HAS_STRICT_OPTION(cx))
+        return JS_TRUE;
+ 
+    va_start(ap, errorNumber);
+    warning = ReportCompileErrorNumber(cx, handle, flags, errorNumber,
+                                       &report, JS_FALSE, ap);
+    va_end(ap);
+
+    if (report.messageArgs)
+        JS_free(cx, (void *)report.messageArgs);
 
-    if (ts && !JSREPORT_IS_WARNING(flags)) {
-        /* Set the error flag to suppress spurious reports. */
-        ts->flags |= TSF_ERROR;
-    }
     return warning;
 }
 
+static JSBool
+GrowStringBuffer(JSStringBuffer *sb, size_t newlength)
+{
+    ptrdiff_t offset;
+    jschar *bp;
+
+    offset = PTRDIFF(sb->ptr, sb->base, jschar);
+    JS_ASSERT(offset >= 0);
+    newlength += offset + 1;
+    if ((size_t)offset < newlength && newlength < ~(size_t)0 / sizeof(jschar))
+        bp = realloc(sb->base, newlength * sizeof(jschar));
+    else
+        bp = NULL;
+    if (!bp) {
+        free(sb->base);
+        sb->base = STRING_BUFFER_ERROR_BASE;
+        return JS_FALSE;
+    }
+    sb->base = bp;
+    sb->ptr = bp + offset;
+    sb->limit = bp + newlength - 1;
+    return JS_TRUE;
+}
+
+static void
+FreeStringBuffer(JSStringBuffer *sb)
+{
+    JS_ASSERT(STRING_BUFFER_OK(sb));
+    if (sb->base)
+        free(sb->base);
+}
+
+void
+js_InitStringBuffer(JSStringBuffer *sb)
+{
+    sb->base = sb->limit = sb->ptr = NULL;
+    sb->data = NULL;
+    sb->grow = GrowStringBuffer;
+    sb->free = FreeStringBuffer;
+}
+
+void
+js_FinishStringBuffer(JSStringBuffer *sb)
+{
+    sb->free(sb);
+}
+
+#define ENSURE_STRING_BUFFER(sb,n) \
+    ((sb)->ptr + (n) <= (sb)->limit || sb->grow(sb, n))
+
+static void
+FastAppendChar(JSStringBuffer *sb, jschar c)
+{
+    if (!STRING_BUFFER_OK(sb))
+        return;
+    if (!ENSURE_STRING_BUFFER(sb, 1))
+        return;
+    *sb->ptr++ = c;
+}
+
+void
+js_AppendChar(JSStringBuffer *sb, jschar c)
+{
+    jschar *bp;
+
+    if (!STRING_BUFFER_OK(sb))
+        return;
+    if (!ENSURE_STRING_BUFFER(sb, 1))
+        return;
+    bp = sb->ptr;
+    *bp++ = c;
+    *bp = 0;
+    sb->ptr = bp;
+}
+
+#if JS_HAS_XML_SUPPORT
+
+void
+js_RepeatChar(JSStringBuffer *sb, jschar c, uintN count)
+{
+    jschar *bp;
+
+    if (!STRING_BUFFER_OK(sb) || count == 0)
+        return;
+    if (!ENSURE_STRING_BUFFER(sb, count))
+        return;
+    for (bp = sb->ptr; count; --count)
+        *bp++ = c;
+    *bp = 0;
+    sb->ptr = bp;
+}
+
+void
+js_AppendCString(JSStringBuffer *sb, const char *asciiz)
+{
+    size_t length;
+    jschar *bp;
+
+    if (!STRING_BUFFER_OK(sb) || *asciiz == '\0')
+        return;
+    length = strlen(asciiz);
+    if (!ENSURE_STRING_BUFFER(sb, length))
+        return;
+    for (bp = sb->ptr; length; --length)
+        *bp++ = (jschar) *asciiz++;
+    *bp = 0;
+    sb->ptr = bp;
+}
+
+void
+js_AppendJSString(JSStringBuffer *sb, JSString *str)
+{
+    size_t length;
+    jschar *bp;
+
+    if (!STRING_BUFFER_OK(sb))
+        return;
+    length = JSSTRING_LENGTH(str);
+    if (length == 0 || !ENSURE_STRING_BUFFER(sb, length))
+        return;
+    bp = sb->ptr;
+    js_strncpy(bp, JSSTRING_CHARS(str), length);
+    bp += length;
+    *bp = 0;
+    sb->ptr = bp;
+}
+
+static JSBool
+GetXMLEntity(JSContext *cx, JSTokenStream *ts)
+{
+    ptrdiff_t offset, length, i;
+    int32 c, d;
+    JSBool ispair;
+    jschar *bp, digit;
+    char *bytes;
+    JSErrNum msg;
+
+    /* Put the entity, including the '&' already scanned, in ts->tokenbuf. */
+    offset = PTRDIFF(ts->tokenbuf.ptr, ts->tokenbuf.base, jschar);
+    FastAppendChar(&ts->tokenbuf, '&');
+    while ((c = GetChar(ts)) != ';') {
+        if (c == EOF || c == '\n') {
+            js_ReportCompileErrorNumber(cx, ts,
+                                        JSREPORT_TS | JSREPORT_ERROR,
+                                        JSMSG_END_OF_XML_ENTITY);
+            return JS_FALSE;
+        }
+        FastAppendChar(&ts->tokenbuf, (jschar) c);
+    }
+
+    /* Let length be the number of jschars after the '&', including the ';'. */
+    length = PTRDIFF(ts->tokenbuf.ptr, ts->tokenbuf.base, jschar) - offset;
+    bp = ts->tokenbuf.base + offset;
+    c = d = 0;
+    ispair = JS_FALSE;
+    if (length > 2 && bp[1] == '#') {
+        /* Match a well-formed XML Character Reference. */
+        i = 2;
+        if (length > 3 && JS_TOLOWER(bp[i]) == 'x') {
+            if (length > 9)     /* at most 6 hex digits allowed */
+                goto badncr;
+            while (++i < length) {
+                digit = bp[i];
+                if (!JS7_ISHEX(digit))
+                    goto badncr;
+                c = (c << 4) + JS7_UNHEX(digit);
+            }
+        } else {
+            while (i < length) {
+                digit = bp[i++];
+                if (!JS7_ISDEC(digit))
+                    goto badncr;
+                c = (c * 10) + JS7_UNDEC(digit);
+                if (c < 0)
+                    goto badncr;
+            }
+        }
+
+        if (0x10000 <= c && c <= 0x10FFFF) {
+            /* Form a surrogate pair (c, d) -- c is the high surrogate. */
+            d = 0xDC00 + (c & 0x3FF);
+            c = 0xD7C0 + (c >> 10);
+            ispair = JS_TRUE;
+        } else {
+            /* Enforce the http://www.w3.org/TR/REC-xml/#wf-Legalchar WFC. */
+            if (c != 0x9 && c != 0xA && c != 0xD &&
+                !(0x20 <= c && c <= 0xD7FF) &&
+                !(0xE000 <= c && c <= 0xFFFD)) {
+                goto badncr;
+            }
+        }
+    } else {
+        /* Try to match one of the five XML 1.0 predefined entities. */
+        switch (length) {
+          case 3:
+            if (bp[2] == 't') {
+                if (bp[1] == 'l')
+                    c = '<';
+                else if (bp[1] == 'g')
+                    c = '>';
+            }
+            break;
+          case 4:
+            if (bp[1] == 'a' && bp[2] == 'm' && bp[3] == 'p')
+                c = '&';
+            break;
+          case 5:
+            if (bp[3] == 'o') {
+                if (bp[1] == 'a' && bp[2] == 'p' && bp[4] == 's')
+                    c = '\'';
+                else if (bp[1] == 'q' && bp[2] == 'u' && bp[4] == 't')
+                    c = '"';
+            }
+            break;
+        }
+        if (c == 0) {
+            msg = JSMSG_UNKNOWN_XML_ENTITY;
+            goto bad;
+        }
+    }
+
+    /* If we matched, retract ts->tokenbuf and store the entity's value. */
+    *bp++ = (jschar) c;
+    if (ispair)
+        *bp++ = (jschar) d;
+    *bp = 0;
+    ts->tokenbuf.ptr = bp;
+    return JS_TRUE;
+
+badncr:
+    msg = JSMSG_BAD_XML_NCR;
+bad:
+    /* No match: throw a TypeError per ECMA-357 10.3.2.1 step 8(a). */
+    bytes = js_DeflateString(cx, bp + 1,
+                             PTRDIFF(ts->tokenbuf.ptr, bp, jschar) - 1);
+    if (bytes) {
+        js_ReportCompileErrorNumber(cx, ts, JSREPORT_TS | JSREPORT_ERROR,
+                                    msg, bytes);
+        JS_free(cx, bytes);
+    }
+    return JS_FALSE;
+}
+
+#endif /* JS_HAS_XML_SUPPORT */
+
 JSTokenType
 js_PeekToken(JSContext *cx, JSTokenStream *ts)
 {
@@ -666,55 +1078,15 @@ js_PeekTokenSameLine(JSContext *cx, JSTokenStream *ts)
     JSTokenType tt;
 
     JS_ASSERT(ts->lookahead == 0 ||
-              ON_CURRENT_LINE(ts, CURRENT_TOKEN(ts).pos));
+              ON_CURRENT_LINE(ts, CURRENT_TOKEN(ts).pos) ||
+              ts->tokens[(ts->cursor + ts->lookahead) & NTOKENS_MASK].type
+                  == TOK_EOL);
     ts->flags |= TSF_NEWLINES;
     tt = js_PeekToken(cx, ts);
     ts->flags &= ~TSF_NEWLINES;
     return tt;
 }
 
-#define TBMIN   64
-
-static JSBool
-GrowTokenBuf(JSContext *cx, JSTokenBuf *tb)
-{
-    jschar *base;
-    ptrdiff_t offset, length;
-    size_t tbsize;
-    JSArenaPool *pool;
-
-    base = tb->base;
-    offset = PTRDIFF(tb->ptr, base, jschar);
-    pool = &cx->tempPool;
-    if (!base) {
-        tbsize = TBMIN * sizeof(jschar);
-        length = TBMIN;
-        JS_ARENA_ALLOCATE_CAST(base, jschar *, pool, tbsize);
-    } else {
-        length = PTRDIFF(tb->limit, base, jschar);
-        tbsize = length * sizeof(jschar);
-        length <<= 1;
-        JS_ARENA_GROW_CAST(base, jschar *, pool, tbsize, tbsize);
-    }
-    if (!base) {
-        JS_ReportOutOfMemory(cx);
-        return JS_FALSE;
-    }
-    tb->base = base;
-    tb->limit = base + length;
-    tb->ptr = base + offset;
-    return JS_TRUE;
-}
-
-static JSBool
-AddToTokenBuf(JSContext *cx, JSTokenBuf *tb, jschar c)
-{
-    if (tb->ptr == tb->limit && !GrowTokenBuf(cx, tb))
-        return JS_FALSE;
-    *tb->ptr++ = c;
-    return JS_TRUE;
-}
-
 /*
  * We have encountered a '\': check for a Unicode escape sequence after it,
  * returning the character code value if we found a Unicode escape sequence.
@@ -741,14 +1113,16 @@ GetUnicodeEscape(JSTokenStream *ts)
 }
 
 static JSToken *
-NewToken(JSTokenStream *ts)
+NewToken(JSTokenStream *ts, ptrdiff_t adjust)
 {
     JSToken *tp;
 
     ts->cursor = (ts->cursor + 1) & NTOKENS_MASK;
     tp = &CURRENT_TOKEN(ts);
-    tp->ptr = ts->linebuf.ptr - 1;
-    tp->pos.begin.index = ts->linepos + (tp->ptr - ts->linebuf.base);
+    tp->ptr = ts->linebuf.ptr + adjust;
+    tp->pos.begin.index = ts->linepos +
+                          PTRDIFF(tp->ptr, ts->linebuf.base, jschar) -
+                          ts->ungetpos;
     tp->pos.begin.lineno = tp->pos.end.lineno = (uint16)ts->lineno;
     return tp;
 }
@@ -763,20 +1137,21 @@ js_GetToken(JSContext *cx, JSTokenStream *ts)
     JSBool hadUnicodeEscape;
 
 #define INIT_TOKENBUF()     (ts->tokenbuf.ptr = ts->tokenbuf.base)
-#define TRIM_TOKENBUF(i)    (ts->tokenbuf.ptr = ts->tokenbuf.base + i)
-#define TOKENBUF_LENGTH()   (ts->tokenbuf.ptr - ts->tokenbuf.base)
+#define TOKENBUF_LENGTH()   PTRDIFF(ts->tokenbuf.ptr, ts->tokenbuf.base, jschar)
+#define TOKENBUF_OK()       STRING_BUFFER_OK(&ts->tokenbuf)
+#define TOKENBUF_TO_ATOM()  (TOKENBUF_OK()                                    \
+                             ? js_AtomizeChars(cx,                            \
+                                               TOKENBUF_BASE(),               \
+                                               TOKENBUF_LENGTH(),             \
+                                               0)                             \
+                             : NULL)
+#define ADD_TO_TOKENBUF(c)  FastAppendChar(&ts->tokenbuf, (jschar) (c))
+
+/* The following 4 macros should only be used when TOKENBUF_OK() is true. */
 #define TOKENBUF_BASE()     (ts->tokenbuf.base)
 #define TOKENBUF_CHAR(i)    (ts->tokenbuf.base[i])
-#define TOKENBUF_TO_ATOM()  (js_AtomizeChars(cx,                              \
-                                             TOKENBUF_BASE(),                 \
-                                             TOKENBUF_LENGTH(),               \
-                                             0))
-
-#define ADD_TO_TOKENBUF(c)                                                    \
-    JS_BEGIN_MACRO                                                            \
-        if (!AddToTokenBuf(cx, &ts->tokenbuf, (jschar)c))                     \
-            goto error;                                                       \
-    JS_END_MACRO
+#define TRIM_TOKENBUF(i)    (ts->tokenbuf.ptr = ts->tokenbuf.base + i)
+#define NUL_TERM_TOKENBUF() (*ts->tokenbuf.ptr = 0)
 
     /* If there was a fatal error, keep returning TOK_ERROR. */
     if (ts->flags & TSF_ERROR)
@@ -784,6 +1159,7 @@ js_GetToken(JSContext *cx, JSTokenStream *ts)
 
     /* Check for a pushed-back token resulting from mismatching lookahead. */
     while (ts->lookahead != 0) {
+        JS_ASSERT(!(ts->flags & TSF_XMLTEXTMODE));
         ts->lookahead--;
         ts->cursor = (ts->cursor + 1) & NTOKENS_MASK;
         tt = CURRENT_TOKEN(ts).type;
@@ -791,6 +1167,163 @@ js_GetToken(JSContext *cx, JSTokenStream *ts)
             return tt;
     }
 
+#if JS_HAS_XML_SUPPORT
+    if (ts->flags & TSF_XMLTEXTMODE) {
+        tt = TOK_XMLSPACE;      /* veto if non-space, return TOK_XMLTEXT */
+        tp = NewToken(ts, 0);
+        INIT_TOKENBUF();
+        qc = (ts->flags & TSF_XMLONLYMODE) ? '<' : '{';
+
+        while ((c = GetChar(ts)) != qc && c != '<' && c != EOF) {
+            if (c == '&' && qc == '<') {
+                if (!GetXMLEntity(cx, ts))
+                    goto error;
+                tt = TOK_XMLTEXT;
+                continue;
+            }
+
+            if (!JS_ISXMLSPACE(c))
+                tt = TOK_XMLTEXT;
+            ADD_TO_TOKENBUF(c);
+        }
+        UngetChar(ts, c);
+
+        if (TOKENBUF_LENGTH() == 0) {
+            atom = NULL;
+        } else {
+            atom = TOKENBUF_TO_ATOM();
+            if (!atom)
+                goto error;
+        }
+        tp->pos.end.lineno = (uint16)ts->lineno;
+        tp->t_op = JSOP_STRING;
+        tp->t_atom = atom;
+        goto out;
+    }
+
+    if (ts->flags & TSF_XMLTAGMODE) {
+        tp = NewToken(ts, 0);
+        c = GetChar(ts);
+        if (JS_ISXMLSPACE(c)) {
+            do {
+                c = GetChar(ts);
+            } while (JS_ISXMLSPACE(c));
+            UngetChar(ts, c);
+            tt = TOK_XMLSPACE;
+            goto out;
+        }
+
+        if (c == EOF) {
+            tt = TOK_EOF;
+            goto out;
+        }
+
+        INIT_TOKENBUF();
+        if (JS_ISXMLNSSTART(c)) {
+            JSBool sawColon = JS_FALSE;
+
+            ADD_TO_TOKENBUF(c);
+            while ((c = GetChar(ts)) != EOF && JS_ISXMLNAME(c)) {
+                if (c == ':') {
+                    int nextc;
+
+                    if (sawColon ||
+                        (nextc = PeekChar(ts),
+                         ((ts->flags & TSF_XMLONLYMODE) || nextc != '{') &&
+                         !JS_ISXMLNAME(nextc))) {
+                        js_ReportCompileErrorNumber(cx, ts,
+                                                    JSREPORT_TS |
+                                                    JSREPORT_ERROR,
+                                                    JSMSG_BAD_XML_QNAME);
+                        goto error;
+                    }
+                    sawColon = JS_TRUE;
+                }
+
+                ADD_TO_TOKENBUF(c);
+            }
+
+            UngetChar(ts, c);
+            atom = TOKENBUF_TO_ATOM();
+            if (!atom)
+                goto error;
+            tp->t_op = JSOP_STRING;
+            tp->t_atom = atom;
+            tt = TOK_XMLNAME;
+            goto out;
+        }
+
+        switch (c) {
+          case '{':
+            if (ts->flags & TSF_XMLONLYMODE)
+                goto bad_xml_char;
+            tt = TOK_LC;
+            goto out;
+
+          case '=':
+            tt = TOK_ASSIGN;
+            goto out;
+
+          case '"':
+          case '\'':
+            qc = c;
+            while ((c = GetChar(ts)) != qc) {
+                if (c == EOF) {
+                    js_ReportCompileErrorNumber(cx, ts,
+                                                JSREPORT_TS | JSREPORT_ERROR,
+                                                JSMSG_UNTERMINATED_STRING);
+                    goto error;
+                }
+
+                /*
+                 * XML attribute values are double-quoted when pretty-printed,
+                 * so escape " if it is expressed directly in a single-quoted
+                 * attribute value.
+                 */
+                if (c == '"' && !(ts->flags & TSF_XMLONLYMODE)) {
+                    JS_ASSERT(qc == '\'');
+                    js_AppendCString(&ts->tokenbuf, js_quot_entity_str);
+                    continue;
+                }
+
+                if (c == '&' && (ts->flags & TSF_XMLONLYMODE)) {
+                    if (!GetXMLEntity(cx, ts))
+                        goto error;
+                    continue;
+                }
+
+                ADD_TO_TOKENBUF(c);
+            }
+            atom = TOKENBUF_TO_ATOM();
+            if (!atom)
+                goto error;
+            tp->pos.end.lineno = (uint16)ts->lineno;
+            tp->t_op = JSOP_STRING;
+            tp->t_atom = atom;
+            tt = TOK_XMLATTR;
+            goto out;
+
+          case '>':
+            tt = TOK_XMLTAGC;
+            goto out;
+
+          case '/':
+            if (MatchChar(ts, '>')) {
+                tt = TOK_XMLPTAGC;
+                goto out;
+            }
+            /* FALL THROUGH */
+
+          bad_xml_char:
+          default:
+            js_ReportCompileErrorNumber(cx, ts, JSREPORT_TS | JSREPORT_ERROR,
+                                        JSMSG_BAD_XML_CHARACTER);
+            goto error;
+        }
+        /* NOTREACHED */
+    }
+#endif /* JS_HAS_XML_SUPPORT */
+
 retry:
     do {
         c = GetChar(ts);
@@ -801,20 +1334,17 @@ retry:
         }
     } while (JS_ISSPACE(c));
 
-    tp = NewToken(ts);
+    tp = NewToken(ts, -1);
     if (c == EOF) {
         tt = TOK_EOF;
         goto out;
     }
 
-    if (c != '-' && c != '\n')
-        ts->flags |= TSF_DIRTYLINE;
-
     hadUnicodeEscape = JS_FALSE;
-    if (JS_ISIDENT_START(c) ||
+    if (JS_ISIDSTART(c) ||
         (c == '\\' &&
          (c = GetUnicodeEscape(ts),
-          hadUnicodeEscape = JS_ISIDENT_START(c)))) {
+          hadUnicodeEscape = JS_ISIDSTART(c)))) {
         INIT_TOKENBUF();
         for (;;) {
             ADD_TO_TOKENBUF(c);
@@ -835,11 +1365,28 @@ retry:
         if (!atom)
             goto error;
         if (!hadUnicodeEscape && ATOM_KEYWORD(atom)) {
-            struct keyword *kw = ATOM_KEYWORD(atom);
-
-            if (JSVERSION_IS_ECMA(cx->version) || kw->version <= cx->version) {
-                tp->t_op = (JSOp) kw->op;
+            struct keyword *kw;
+            
+            JS_ASSERT(!(atom->flags & ATOM_HIDDEN));
+            kw = ATOM_KEYWORD(atom);
+            if (kw->tokentype == TOK_RESERVED) {
+                char buf[MAX_KEYWORD_LENGTH + 1];
+                size_t buflen = sizeof(buf) - 1;
+                if (!js_DeflateStringToBuffer(cx, TOKENBUF_BASE(), TOKENBUF_LENGTH(),
+                                                  buf, &buflen))
+                    goto error;
+                buf [buflen] = 0;
+                if (!js_ReportCompileErrorNumber(cx, ts,
+                                                 JSREPORT_TS |
+                                                 JSREPORT_WARNING |
+                                                 JSREPORT_STRICT,
+                                                 JSMSG_RESERVED_ID, buf)) {
+                    goto error;
+                }
+            } else if (JS_VERSION_IS_ECMA(cx) ||
+                       kw->version <= (cx->version & JSVERSION_MASK)) {
                 tt = kw->tokentype;
+                tp->t_op = (JSOp) kw->op;
                 goto out;
             }
         }
@@ -880,7 +1427,8 @@ retry:
                  * not always be so permissive, so we warn about it.
                  */
                 if (radix == 8 && c >= '8') {
-                    if (!js_ReportCompileErrorNumber(cx, ts, NULL,
+                    if (!js_ReportCompileErrorNumber(cx, ts,
+                                                     JSREPORT_TS |
                                                      JSREPORT_WARNING,
                                                      JSMSG_BAD_OCTAL,
                                                      c == '8' ? "08" : "09")) {
@@ -908,7 +1456,8 @@ retry:
                     c = GetChar(ts);
                 }
                 if (!JS7_ISDEC(c)) {
-                    js_ReportCompileErrorNumber(cx, ts, NULL, JSREPORT_ERROR,
+                    js_ReportCompileErrorNumber(cx, ts,
+                                                JSREPORT_TS | JSREPORT_ERROR,
                                                 JSMSG_MISSING_EXPONENT);
                     goto error;
                 }
@@ -923,15 +1472,19 @@ retry:
         UngetChar(ts, c);
         ADD_TO_TOKENBUF(0);
 
+        if (!TOKENBUF_OK())
+            goto error;
         if (radix == 10) {
             if (!js_strtod(cx, TOKENBUF_BASE(), &endptr, &dval)) {
-                js_ReportCompileErrorNumber(cx, ts, NULL, JSREPORT_ERROR,
+                js_ReportCompileErrorNumber(cx, ts,
+                                            JSREPORT_TS | JSREPORT_ERROR,
                                             JSMSG_OUT_OF_MEMORY);
                 goto error;
             }
         } else {
             if (!js_strtointeger(cx, TOKENBUF_BASE(), &endptr, radix, &dval)) {
-                js_ReportCompileErrorNumber(cx, ts, NULL, JSREPORT_ERROR,
+                js_ReportCompileErrorNumber(cx, ts,
+                                            JSREPORT_TS | JSREPORT_ERROR,
                                             JSMSG_OUT_OF_MEMORY);
                 goto error;
             }
@@ -947,7 +1500,8 @@ retry:
         while ((c = GetChar(ts)) != qc) {
             if (c == '\n' || c == EOF) {
                 UngetChar(ts, c);
-                js_ReportCompileErrorNumber(cx, ts, NULL, JSREPORT_ERROR,
+                js_ReportCompileErrorNumber(cx, ts,
+                                            JSREPORT_TS | JSREPORT_ERROR,
                                             JSMSG_UNTERMINATED_STRING);
                 goto error;
             }
@@ -998,7 +1552,7 @@ retry:
                             c = (JS7_UNHEX(cp[0]) << 4) + JS7_UNHEX(cp[1]);
                             SkipChars(ts, 2);
                         }
-                    } else if (c == '\n' && JSVERSION_IS_ECMA(cx->version)) {
+                    } else if (c == '\n' && JS_VERSION_IS_ECMA(cx)) {
                         /* ECMA follows C by removing escaped newlines. */
                         continue;
                     }
@@ -1018,25 +1572,33 @@ retry:
     }
 
     switch (c) {
-      case '\n':
-        tt = TOK_EOL;
-        break;
-
-      case ';': tt = TOK_SEMI; break;
-      case '[': tt = TOK_LB; break;
-      case ']': tt = TOK_RB; break;
-      case '{': tt = TOK_LC; break;
-      case '}': tt = TOK_RC; break;
-      case '(': tt = TOK_LP; break;
-      case ')': tt = TOK_RP; break;
-      case ',': tt = TOK_COMMA; break;
-      case '?': tt = TOK_HOOK; break;
+      case '\n': tt = TOK_EOL; goto eol_out;
+      case ';':  tt = TOK_SEMI; break;
+      case '[':  tt = TOK_LB; break;
+      case ']':  tt = TOK_RB; break;
+      case '{':  tt = TOK_LC; break;
+      case '}':  tt = TOK_RC; break;
+      case '(':  tt = TOK_LP; break;
+      case ')':  tt = TOK_RP; break;
+      case ',':  tt = TOK_COMMA; break;
+      case '?':  tt = TOK_HOOK; break;
 
       case '.':
-        tt = TOK_DOT;
+#if JS_HAS_XML_SUPPORT
+        if (MatchChar(ts, c))
+            tt = TOK_DBLDOT;
+        else
+#endif
+            tt = TOK_DOT;
         break;
 
       case ':':
+#if JS_HAS_XML_SUPPORT
+        if (MatchChar(ts, c)) {
+            tt = TOK_DBLCOLON;
+            break;
+        }
+#endif
         /*
          * Default so compiler can modify to JSOP_GETTER if 'p getter: v' in an
          * object initializer, likewise for setter.
@@ -1104,12 +1666,156 @@ retry:
         }
         break;
 
+#if JS_HAS_XML_SUPPORT
+      case '@':
+        tt = TOK_AT;
+        break;
+#endif
+
       case '<':
+#if JS_HAS_XML_SUPPORT
+        /*
+         * After much testing, it's clear that Postel's advice to protocol
+         * designers ("be liberal in what you accept, and conservative in what
+         * you send") invites a natural-law repercussion for JS as "protocol":
+         *
+         * "If you are liberal in what you accept, others will utterly fail to
+         *  be conservative in what they send."
+         *
+         * Which means you will get <!-- comments to end of line in the middle
+         * of .js files, and after if conditions whose then statements are on
+         * the next line, and other wonders.  See at least the following bugs:
+         * https://bugzilla.mozilla.org/show_bug.cgi?id=309242
+         * https://bugzilla.mozilla.org/show_bug.cgi?id=309712
+         * https://bugzilla.mozilla.org/show_bug.cgi?id=310993
+         *
+         * So without JSOPTION_XML, we never scan an XML comment or CDATA
+         * literal.  We always scan <! as the start of an HTML comment hack
+         * to end of line, used since Netscape 2 to hide script tag content
+         * from script-unaware browsers.
+         */
+        if ((ts->flags & TSF_OPERAND) &&
+            (JS_HAS_XML_OPTION(cx) || PeekChar(ts) != '!')) {
+            /* Check for XML comment or CDATA section. */
+            if (MatchChar(ts, '!')) {
+                INIT_TOKENBUF();
+
+                /* Scan XML comment. */
+                if (MatchChar(ts, '-')) {
+                    if (!MatchChar(ts, '-'))
+                        goto bad_xml_markup;
+                    while ((c = GetChar(ts)) != '-' || !MatchChar(ts, '-')) {
+                        if (c == EOF)
+                            goto bad_xml_markup;
+                        ADD_TO_TOKENBUF(c);
+                    }
+                    tt = TOK_XMLCOMMENT;
+                    tp->t_op = JSOP_XMLCOMMENT;
+                    goto finish_xml_markup;
+                }
+
+                /* Scan CDATA section. */
+                if (MatchChar(ts, '[')) {
+                    jschar cp[6];
+                    if (PeekChars(ts, 6, cp) &&
+                        cp[0] == 'C' &&
+                        cp[1] == 'D' &&
+                        cp[2] == 'A' &&
+                        cp[3] == 'T' &&
+                        cp[4] == 'A' &&
+                        cp[5] == '[') {
+                        SkipChars(ts, 6);
+                        while ((c = GetChar(ts)) != ']' ||
+                               !PeekChars(ts, 2, cp) ||
+                               cp[0] != ']' ||
+                               cp[1] != '>') {
+                            if (c == EOF)
+                                goto bad_xml_markup;
+                            ADD_TO_TOKENBUF(c);
+                        }
+                        GetChar(ts);            /* discard ] but not > */
+                        tt = TOK_XMLCDATA;
+                        tp->t_op = JSOP_XMLCDATA;
+                        goto finish_xml_markup;
+                    }
+                    goto bad_xml_markup;
+                }
+            }
+
+            /* Check for processing instruction. */
+            if (MatchChar(ts, '?')) {
+                JSBool inTarget = JS_TRUE;
+                size_t targetLength = 0;
+                ptrdiff_t contentIndex = -1;
+
+                INIT_TOKENBUF();
+                while ((c = GetChar(ts)) != '?' || PeekChar(ts) != '>') {
+                    if (c == EOF)
+                        goto bad_xml_markup;
+                    if (inTarget) {
+                        if (JS_ISXMLSPACE(c)) {
+                            if (TOKENBUF_LENGTH() == 0)
+                                goto bad_xml_markup;
+                            inTarget = JS_FALSE;
+                        } else {
+                            if (!((TOKENBUF_LENGTH() == 0)
+                                  ? JS_ISXMLNSSTART(c)
+                                  : JS_ISXMLNS(c))) {
+                                goto bad_xml_markup;
+                            }
+                            ++targetLength;
+                        }
+                    } else {
+                        if (contentIndex < 0 && !JS_ISXMLSPACE(c))
+                            contentIndex = TOKENBUF_LENGTH();
+                    }
+                    ADD_TO_TOKENBUF(c);
+                }
+                if (contentIndex < 0) {
+                    atom = cx->runtime->atomState.emptyAtom;
+                } else {
+                    if (!TOKENBUF_OK())
+                        goto error;
+                    atom = js_AtomizeChars(cx,
+                                           &TOKENBUF_CHAR(contentIndex),
+                                           TOKENBUF_LENGTH() - contentIndex,
+                                           0);
+                    if (!atom)
+                        goto error;
+                    TRIM_TOKENBUF(targetLength);
+                }
+                tp->t_atom2 = atom;
+                tt = TOK_XMLPI;
+
+        finish_xml_markup:
+                if (!MatchChar(ts, '>'))
+                    goto bad_xml_markup;
+                atom = TOKENBUF_TO_ATOM();
+                if (!atom)
+                    goto error;
+                tp->t_atom = atom;
+                tp->pos.end.lineno = (uint16)ts->lineno;
+                goto out;
+            }
+
+            /* An XML start-of-tag character. */
+            tt = MatchChar(ts, '/') ? TOK_XMLETAGO : TOK_XMLSTAGO;
+            goto out;
+
+        bad_xml_markup:
+            js_ReportCompileErrorNumber(cx, ts, JSREPORT_TS | JSREPORT_ERROR,
+                                        JSMSG_BAD_XML_MARKUP);
+            goto error;
+        }
+#endif /* JS_HAS_XML_SUPPORT */
+
         /* NB: treat HTML begin-comment as comment-till-end-of-line */
         if (MatchChar(ts, '!')) {
             if (MatchChar(ts, '-')) {
-                if (MatchChar(ts, '-'))
+                if (MatchChar(ts, '-')) {
+                    ts->flags |= TSF_IN_HTML_COMMENT;
                     goto skipline;
+                }
                 UngetChar(ts, '-');
             }
             UngetChar(ts, '!');
@@ -1207,21 +1913,33 @@ retry:
             }
 
 skipline:
-            while ((c = GetChar(ts)) != EOF && c != '\n')
-                continue;
+            /* Optimize line skipping if we are not in an HTML comment. */
+            if (ts->flags & TSF_IN_HTML_COMMENT) {
+                while ((c = GetChar(ts)) != EOF && c != '\n') {
+                    if (c == '-' && MatchChar(ts, '-') && MatchChar(ts, '>'))
+                        ts->flags &= ~TSF_IN_HTML_COMMENT;
+                }
+            } else {
+                while ((c = GetChar(ts)) != EOF && c != '\n')
+                    continue;
+            }
             UngetChar(ts, c);
+            ts->cursor = (ts->cursor - 1) & NTOKENS_MASK;
             goto retry;
         }
+
         if (MatchChar(ts, '*')) {
             while ((c = GetChar(ts)) != EOF &&
                    !(c == '*' && MatchChar(ts, '/'))) {
                 /* Ignore all characters until comment close. */
             }
             if (c == EOF) {
-                js_ReportCompileErrorNumber(cx, ts, NULL, JSREPORT_ERROR,
+                js_ReportCompileErrorNumber(cx, ts,
+                                            JSREPORT_TS | JSREPORT_ERROR,
                                             JSMSG_UNTERMINATED_COMMENT);
                 goto error;
             }
+            ts->cursor = (ts->cursor - 1) & NTOKENS_MASK;
             goto retry;
         }
 
@@ -1229,18 +1947,28 @@ skipline:
         if (ts->flags & TSF_OPERAND) {
             JSObject *obj;
             uintN flags;
+            JSBool inCharClass = JS_FALSE;
 
             INIT_TOKENBUF();
-            while ((c = GetChar(ts)) != '/') {
+            for (;;) {
+                c = GetChar(ts);
                 if (c == '\n' || c == EOF) {
                     UngetChar(ts, c);
-                    js_ReportCompileErrorNumber(cx, ts, NULL, JSREPORT_ERROR,
+                    js_ReportCompileErrorNumber(cx, ts,
+                                                JSREPORT_TS | JSREPORT_ERROR,
                                                 JSMSG_UNTERMINATED_REGEXP);
                     goto error;
                 }
                 if (c == '\\') {
                     ADD_TO_TOKENBUF(c);
                     c = GetChar(ts);
+                } else if (c == '[') {
+                    inCharClass = JS_TRUE;
+                } else if (c == ']') {
+                    inCharClass = JS_FALSE;
+                } else if (c == '/' && !inCharClass) {
+                    /* For compat with IE, allow unescaped / in char classes. */
+                    break;
                 }
                 ADD_TO_TOKENBUF(c);
             }
@@ -1257,11 +1985,16 @@ skipline:
             c = PeekChar(ts);
             if (JS7_ISLET(c)) {
                 tp->ptr = ts->linebuf.ptr - 1;
-                js_ReportCompileErrorNumber(cx, ts, NULL, JSREPORT_ERROR,
+                js_ReportCompileErrorNumber(cx, ts,
+                                            JSREPORT_TS | JSREPORT_ERROR,
                                             JSMSG_BAD_REGEXP_FLAG);
                 (void) GetChar(ts);
                 goto error;
             }
+            /* XXXbe fix jsregexp.c so it doesn't depend on NUL termination */
+            if (!TOKENBUF_OK())
+                goto error;
+            NUL_TERM_TOKENBUF();
             obj = js_NewRegExpObject(cx, ts,
                                      TOKENBUF_BASE(),
                                      TOKENBUF_LENGTH(),
@@ -1319,14 +2052,15 @@ skipline:
             tp->t_op = JSOP_SUB;
             tt = TOK_ASSIGN;
         } else if (MatchChar(ts, c)) {
-            if (PeekChar(ts) == '>' && !(ts->flags & TSF_DIRTYLINE))
+            if (PeekChar(ts) == '>' && !(ts->flags & TSF_DIRTYLINE)) {
+                ts->flags &= ~TSF_IN_HTML_COMMENT;
                 goto skipline;
+            }
             tt = TOK_DEC;
         } else {
             tp->t_op = JSOP_NEG;
             tt = TOK_MINUS;
         }
-        ts->flags |= TSF_DIRTYLINE;
         break;
 
 #if JS_HAS_SHARP_VARS
@@ -1346,7 +2080,8 @@ skipline:
                 break;
             n = 10 * n + JS7_UNDEC(c);
             if (n >= ATOM_INDEX_LIMIT) {
-                js_ReportCompileErrorNumber(cx, ts, NULL, JSREPORT_ERROR,
+                js_ReportCompileErrorNumber(cx, ts,
+                                            JSREPORT_TS | JSREPORT_ERROR,
                                             JSMSG_SHARPVAR_TOO_BIG);
                 goto error;
             }
@@ -1356,7 +2091,8 @@ skipline:
             (c == '=' || c == '#')) {
             char buf[20];
             JS_snprintf(buf, sizeof buf, "#%u%c", n, c);
-            if (!js_ReportCompileErrorNumber(cx, ts, NULL,
+            if (!js_ReportCompileErrorNumber(cx, ts,
+                                             JSREPORT_TS |
                                              JSREPORT_WARNING |
                                              JSREPORT_STRICT,
                                              JSMSG_DEPRECATED_USAGE,
@@ -1372,20 +2108,28 @@ skipline:
             goto badchar;
         break;
       }
+#endif /* JS_HAS_SHARP_VARS */
 
+#if JS_HAS_SHARP_VARS || JS_HAS_XML_SUPPORT
       badchar:
-#endif /* JS_HAS_SHARP_VARS */
+#endif
 
       default:
-        js_ReportCompileErrorNumber(cx, ts, NULL, JSREPORT_ERROR,
+        js_ReportCompileErrorNumber(cx, ts, JSREPORT_TS | JSREPORT_ERROR,
                                     JSMSG_ILLEGAL_CHARACTER);
         goto error;
     }
 
 out:
+    JS_ASSERT(tt != TOK_EOL);
+    ts->flags |= TSF_DIRTYLINE;
+
+eol_out:
+    if (!STRING_BUFFER_OK(&ts->tokenbuf))
+        tt = TOK_ERROR;
     JS_ASSERT(tt < TOK_LIMIT);
     tp->pos.end.index = ts->linepos +
-                        (ts->linebuf.ptr - ts->linebuf.base) -
+                        PTRDIFF(ts->linebuf.ptr, ts->linebuf.base, jschar) -
                         ts->ungetpos;
     tp->type = tt;
     return tt;
@@ -1396,12 +2140,14 @@ error:
     goto out;
 
 #undef INIT_TOKENBUF
-#undef TRIM_TOKENBUF
 #undef TOKENBUF_LENGTH
-#undef TOKENBUF_BASE
-#undef TOKENBUF_CHAR
+#undef TOKENBUF_OK
 #undef TOKENBUF_TO_ATOM
 #undef ADD_TO_TOKENBUF
+#undef TOKENBUF_BASE
+#undef TOKENBUF_CHAR
+#undef TRIM_TOKENBUF
+#undef NUL_TERM_TOKENBUF
 }
 
 void