mirror of
https://github.com/tesseract-ocr/tesseract.git
synced 2024-12-03 00:49:01 +08:00
068d43d3d8
* Remove old code for string class (no longer needed) Signed-off-by: Stefan Weil <sw@weilnetz.de> * Add std namespace to string class Signed-off-by: Stefan Weil <sw@weilnetz.de>
104 lines
3.8 KiB
C++
104 lines
3.8 KiB
C++
/**********************************************************************
|
|
* File: normstrngs.h
|
|
* Description: Utilities to normalize and manipulate UTF-32 and
|
|
* UTF-8 strings.
|
|
* Author: Ranjith Unnikrishnan
|
|
* Created: Thu July 4 2013
|
|
*
|
|
* (C) Copyright 2013, Google Inc.
|
|
* Licensed under the Apache License, Version 2.0 (the "License");
|
|
* you may not use this file except in compliance with the License.
|
|
* You may obtain a copy of the License at
|
|
* http://www.apache.org/licenses/LICENSE-2.0
|
|
* Unless required by applicable law or agreed to in writing, software
|
|
* distributed under the License is distributed on an "AS IS" BASIS,
|
|
* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
|
|
* See the License for the specific language governing permissions and
|
|
* limitations under the License.
|
|
*
|
|
**********************************************************************/
|
|
|
|
#ifndef TESSERACT_CCUTIL_NORMSTRNGS_H_
|
|
#define TESSERACT_CCUTIL_NORMSTRNGS_H_
|
|
|
|
#include <string>
|
|
#include <vector>
|
|
|
|
#include "validator.h"
|
|
|
|
namespace tesseract {
|
|
|
|
// The standard unicode normalizations.
|
|
enum class UnicodeNormMode {
|
|
kNFD,
|
|
kNFC,
|
|
kNFKD,
|
|
kNFKC,
|
|
};
|
|
|
|
// To normalize away differences in punctuation that are ambiguous, like
|
|
// curly quotes and different widths of dash.
|
|
enum class OCRNorm {
|
|
kNone,
|
|
kNormalize,
|
|
};
|
|
|
|
// To validate and normalize away some subtle differences that can occur in
|
|
// Indic scripts, eg ensuring that an explicit virama is always followed by
|
|
// a zero-width non-joiner.
|
|
enum class GraphemeNorm {
|
|
kNone,
|
|
kNormalize,
|
|
};
|
|
|
|
// Normalizes a UTF8 string according to the given modes. Returns true on
|
|
// success. If false is returned, some failure or invalidity was present, and
|
|
// the result string is produced on a "best effort" basis.
|
|
bool NormalizeUTF8String(UnicodeNormMode u_mode, OCRNorm ocr_normalize,
|
|
GraphemeNorm grapheme_normalize, const char* str8,
|
|
std::string* normalized);
|
|
// Normalizes a UTF8 string according to the given modes and splits into
|
|
// graphemes according to g_mode. Returns true on success. If false is returned,
|
|
// some failure or invalidity was present, and the result string is produced on
|
|
// a "best effort" basis.
|
|
bool NormalizeCleanAndSegmentUTF8(UnicodeNormMode u_mode, OCRNorm ocr_normalize,
|
|
GraphemeNormMode g_mode, bool report_errors,
|
|
const char* str8,
|
|
std::vector<std::string>* graphemes);
|
|
|
|
// Applies just the OCR-specific normalizations and return the normalized char.
|
|
char32 OCRNormalize(char32 ch);
|
|
|
|
// Returns true if the OCRNormalized ch1 and ch2 are the same.
|
|
bool IsOCREquivalent(char32 ch1, char32 ch2);
|
|
|
|
// Returns true if the value lies in the range of valid unicodes.
|
|
bool IsValidCodepoint(const char32 ch);
|
|
|
|
// Returns true a code point has the White_Space Unicode property.
|
|
bool IsWhitespace(const char32 ch);
|
|
// Returns true if every char in the given (null-terminated) string has the
|
|
// White_Space Unicode property.
|
|
bool IsUTF8Whitespace(const char* text);
|
|
|
|
// Returns the length of bytes of the prefix of 'text' that have the White_Space
|
|
// unicode property.
|
|
unsigned int SpanUTF8Whitespace(const char* text);
|
|
|
|
// Returns the length of bytes of the prefix of 'text' that DO NOT have the
|
|
// White_Space unicode property.
|
|
unsigned int SpanUTF8NotWhitespace(const char* text);
|
|
|
|
// Returns true if the char is interchange valid i.e. no C0 or C1 control codes
|
|
// (other than CR LF HT FF) and no non-characters.
|
|
bool IsInterchangeValid(const char32 ch);
|
|
// Same as above but restricted to 7-bit ASCII.
|
|
bool IsInterchangeValid7BitAscii(const char32 ch);
|
|
|
|
// Convert a full-width UTF-8 string to half-width.
|
|
char32 FullwidthToHalfwidth(const char32 ch);
|
|
|
|
} // namespace tesseract
|
|
|
|
#endif // TESSERACT_CCUTIL_NORMSTRNGS_H_
|