Added support for full PC-style keyboards.

BREAKING CHANGE: Redesigned the key character map format to
accomodate full keyboards with more comprehensive suite of modifiers.
Old key character maps will not work anymore and must be updated.
The new format is plain text only and it not compiled to a binary
file (so the "kcm" tool will be removed in a subsequent check-in).

Added FULL keyboard type to support full PC-style keyboards.

Added SPECIAL_FUNCTION keyboard type to support special function
keypads that do not have any printable keys suitable for typing
and only have keys like HOME and POWER

Added a special VIRTUAL_KEYBOARD device id convention that maps
to a virtual keyboard with a fixed known layout.  This is designed
to work around issues injecting input events on devices whose
built-in keyboard does not have a useful key character map (ie.
when the built-in keyboard is a special function keyboard only.)

Modified several places where events were being synthesized
to use the virtual keyboard.

Removed support for the "qwerty" default layout.
The new default layout is "Generic".  For the most part "qwerty"
was being used as a backstop in case the built-in keyboard did
not have a key character map (probably because it was a special
function keypad) and the framework needed to be able to inject
key events anyways.  The latter issue is resolved by using the
special VIRTUAL_KEYBOARD device instead of BUILT_IN_KEYBOARD.

Added the concept of a key modifier behavior so that
MetaKeyKeyListener can distinguish between keyboards that use
chorded vs. toggled modifiers.

Wrote more robust key layout and key character map parsers
to enable support for new keyboard features and user installable
key maps.

Fixed a bug in InputReader generating key ups when keys
are released out of sequence.

Updated tons of documentation.

Currently QwertyKeyListener is being used for full keyboards
with autotext and capitalization disabled.  This mostly works
but causes some problems with character pickers, etc.
These issues will be resolved in subsequent changes.

Change-Id: Ica48f6097a551141c215bc0d2c6f7b3fb634d354
diff --git a/libs/utils/Tokenizer.cpp b/libs/utils/Tokenizer.cpp
new file mode 100644
index 0000000..19dadf0
--- /dev/null
+++ b/libs/utils/Tokenizer.cpp
@@ -0,0 +1,150 @@
+/*
+ * Copyright (C) 2010 The Android Open Source Project
+ *
+ * Licensed under the Apache License, Version 2.0 (the "License");
+ * you may not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ *
+ *      http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+#define LOG_TAG "Tokenizer"
+
+#include <stdlib.h>
+#include <unistd.h>
+#include <fcntl.h>
+#include <errno.h>
+#include <sys/types.h>
+#include <sys/stat.h>
+#include <sys/mman.h>
+#include <utils/Log.h>
+#include <utils/Tokenizer.h>
+
+// Enables debug output for the tokenizer.
+#define DEBUG_TOKENIZER 0
+
+
+namespace android {
+
+static inline bool isDelimiter(char ch, const char* delimiters) {
+    return strchr(delimiters, ch) != NULL;
+}
+
+
+Tokenizer::Tokenizer(const String8& filename, const char* buffer, size_t length) :
+        mFilename(filename), mBuffer(buffer), mLength(length),
+        mCurrent(buffer), mLineNumber(1) {
+}
+
+Tokenizer::~Tokenizer() {
+    munmap((void*)mBuffer, mLength);
+}
+
+status_t Tokenizer::open(const String8& filename, Tokenizer** outTokenizer) {
+    *outTokenizer = NULL;
+
+    int result = NO_ERROR;
+    int fd = ::open(filename.string(), O_RDONLY);
+    if (fd < 0) {
+        result = -errno;
+        LOGE("Error opening file '%s', %s.", filename.string(), strerror(errno));
+    } else {
+        struct stat64 stat;
+        if (fstat64(fd, &stat)) {
+            result = -errno;
+            LOGE("Error getting size of file '%s', %s.", filename.string(), strerror(errno));
+        } else {
+            size_t length = size_t(stat.st_size);
+            void* buffer = mmap(NULL, length, PROT_READ, MAP_FILE | MAP_PRIVATE, fd, 0);
+            if (buffer == MAP_FAILED) {
+                result = -errno;
+                LOGE("Error mapping file '%s', %s.", filename.string(), strerror(errno));
+            } else {
+                if (madvise(buffer, length, MADV_SEQUENTIAL)) {
+                    LOGW("Error calling madvise for mmapped file '%s', %s.", filename.string(),
+                            strerror(errno));
+                }
+
+                *outTokenizer = new Tokenizer(filename, static_cast<const char*>(buffer), length);
+                if (!*outTokenizer) {
+                    result = NO_MEMORY;
+                    LOGE("Error allocating tokenizer for file=%s.", filename.string());
+                    munmap(buffer, length);
+                }
+            }
+        }
+        close(fd);
+    }
+    return result;
+}
+
+String8 Tokenizer::getLocation() const {
+    String8 result;
+    result.appendFormat("%s:%d", mFilename.string(), mLineNumber);
+    return result;
+}
+
+String8 Tokenizer::peekRemainderOfLine() const {
+    const char* end = getEnd();
+    const char* eol = mCurrent;
+    while (eol != end) {
+        char ch = *eol;
+        if (ch == '\n') {
+            break;
+        }
+        eol += 1;
+    }
+    return String8(mCurrent, eol - mCurrent);
+}
+
+String8 Tokenizer::nextToken(const char* delimiters) {
+#if DEBUG_TOKENIZER
+    LOGD("nextToken");
+#endif
+    const char* end = getEnd();
+    const char* tokenStart = mCurrent;
+    while (mCurrent != end) {
+        char ch = *mCurrent;
+        if (ch == '\n' || isDelimiter(ch, delimiters)) {
+            break;
+        }
+        mCurrent += 1;
+    }
+    return String8(tokenStart, mCurrent - tokenStart);
+}
+
+void Tokenizer::nextLine() {
+#if DEBUG_TOKENIZER
+    LOGD("nextLine");
+#endif
+    const char* end = getEnd();
+    while (mCurrent != end) {
+        char ch = *(mCurrent++);
+        if (ch == '\n') {
+            mLineNumber += 1;
+            break;
+        }
+    }
+}
+
+void Tokenizer::skipDelimiters(const char* delimiters) {
+#if DEBUG_TOKENIZER
+    LOGD("skipDelimiters");
+#endif
+    const char* end = getEnd();
+    while (mCurrent != end) {
+        char ch = *mCurrent;
+        if (ch == '\n' || !isDelimiter(ch, delimiters)) {
+            break;
+        }
+        mCurrent += 1;
+    }
+}
+
+} // namespace android