Normalizer.java

Go to the documentation of this file.

// (C) 2003 by Dominique Unruh. GPL

// Reference:
// http://www.unicode.org/unicode/reports/tr15/NormalizerBuilder.java
// http://www.unicode.org/unicode/reports/tr15/Normalizer.java

import java.io.*;

public class Normalizer {
    public static final int D = 2;
    public static final int C = 3;
    public static final int KD = 4;
    public static final int KC = 5;

    public static final int NORMAL = 0;
    public static final int COMPATIBILITY = 1;
    public static final int EXCLUDE = 2;

    public static final int MAX_UNICODE = 0xFFFF;

    private static boolean initialized = false;
    private static String decompositions[] = null;
    private static boolean compatibility[] = null;
    private static int canonicalClass[] = null;
    private static char compositions[][] = null;

    public static synchronized void init() throws IOException {
        if (initialized) return;
        decompositions = new String[MAX_UNICODE+1];
        compatibility = new boolean[MAX_UNICODE+1];
        canonicalClass = new int[MAX_UNICODE+1];
        compositions = new char[MAX_UNICODE+1][];
        DataInputStream data = 
            new DataInputStream
            (new BufferedInputStream
             (new Normalizer().getClass().getResourceAsStream("normalizer_data.dat")));
        try {
            while (true) {
                int num = data.readUnsignedShort();
                //System.out.println("init: "+num);
                if (num>0xFFFF) 
                    throw new IllegalArgumentException
                        ("Cannot handle Unicode > U+FFFF");
                String decomposition = data.readUTF();
                if (decomposition.length()==0) decomposition = null;
                decompositions[num] = decomposition;
                canonicalClass[num] = data.readUnsignedByte();
                int t = data.readUnsignedByte();
                compatibility[num] = (t==COMPATIBILITY);
                if (t==NORMAL && decomposition!=null && decomposition.length()>1) {
                    if (decomposition.length()>2)
                        throw new IllegalArgumentException
                            ("decomposition for "+num+" has length "+decomposition.length());
                    char c1 = decomposition.charAt(0);
                    char c2 = decomposition.charAt(1);
                    if (compositions[c1]==null) {
                        compositions[c1] = new char[c2+0x100];
                    } else if (compositions[c1].length<=c2) {
                        char old[] = compositions[c1];
                        compositions[c1] = new char[c2+0x100];
                        System.arraycopy(old,0,compositions[c1],0,old.length);
                    }
                    //System.out.println(c1+"+"+c2+"="+(char)num);
                    compositions[c1][c2] = (char)num;
                }
            }
        } catch (EOFException e) {};
        initialized = true;
        //System.out.println(compositions[97][776]);
    }

    public static String normalize(int form, String string) throws IOException {
        StringBuffer target = new StringBuffer();
        if (normalize(form,string,0,string.length(),target)!=0)
            throw new RuntimeException("Internal Error: normalize did not return 0");
        return target.toString();
    }

    private static int findStarter(String string, int start) {
        //System.out.print("findStarter("+string+","+start+") = ");
        while (start>0) {
            if (canonicalClass[string.charAt(start)]==0) break;
            start--;
        }
        //System.out.println(start);
        return start;
    }

    public static int normalize(int form, String string, int start, int end, 
                                StringBuffer target) throws IOException {
        if (start==end) {
            target.setLength(0); return start; };
        if (!initialized) init();
        start = findStarter(string,start);
        string = string.substring(start,end);
        switch (form) {
        case D:
            decompose(true,string,target);
            break;
        case C:
            decompose(true,string,target);
            compose(target);
            break;
        case KD:
            decompose(false,string,target);
            break;
        case KC:
            decompose(false,string,target);
            compose(target);
            break;
        default:
            throw new IllegalArgumentException
                ("Unknown normalization form "+form);
        }
        return start;
    }

    private static void compose(StringBuffer target) {
        if (target.length()==0) return;
        int starterPos = 0, compPos = 1;
        char starterChar = target.charAt(0);
        int lastClass = canonicalClass[starterChar];
        if (lastClass != 0) lastClass = 256;
        int len = target.length();
        for (int decompPos = 1; decompPos < len; decompPos++) {
            char c = target.charAt(decompPos);
            int cClass = canonicalClass[c];
            char composite = 0;
            if (compositions[starterChar]!=null && compositions[starterChar].length>c)
                composite = compositions[starterChar][c];
            //System.out.println("Starter "+starterChar+", c "+c);
            if (composite!=0 && (lastClass < cClass || lastClass == 0)) {
                target.setCharAt(starterPos,composite);
                starterChar = composite;
            } else {
                if (cClass == 0) {
                    starterPos = compPos;
                    starterChar = c;
                }
                lastClass = cClass;
                target.setCharAt(compPos++,c);
            }
        }
        target.setLength(compPos);
        return;
    }
    
    private static void decomposeChar(boolean canonical,
                                      char source, StringBuffer target) {
        String decomposition = decompositions[source];
        if (decomposition != null && !(canonical && compatibility[source])) {
            int len = decomposition.length();
            for (int i=0; i<len; i++)
                decomposeChar(canonical,decomposition.charAt(i),target);
        } else {
            target.append(source);
        }
    }

    private static void decompose(boolean canonical, 
                                  String source, StringBuffer target) {
        int sourceLen = source.length();
        StringBuffer buff = new StringBuffer();
        for (int i=0; i<sourceLen; i++) {
            buff.setLength(0);
            decomposeChar(canonical,source.charAt(i),buff);
            int buffLen = buff.length();
            for (int j=0; j<buff.length(); j++) {
                char c = buff.charAt(j);
                int cl = canonicalClass[c];
                int pos = target.length();
                if (cl!=0) {
                    for (; pos>0; pos--) {
                        if (canonicalClass[target.charAt(pos-1)] <= cl)
                            break;
                    }
                }
                target.insert(pos,c);
            }
        }
    }

    public static void main(String args[]) throws IOException {
        StringBuffer arg = new StringBuffer();
        for (int i=0; i<args.length; i++) {
            if (i>0) arg.append(" ");
            arg.append(args[i]);
        }
        System.out.println("String: "+arg);
        System.out.println("Normal: "+normalize(C,arg.toString()));
    }

    public static boolean isCombining(char c) throws IOException {
        if (!initialized) init();
        //System.out.println("isCombining("+c+") = "+(canonicalClass[c]!=0)+" ("+canonicalClass[c]+")");
        return canonicalClass[c]!=0;
    }

    public static boolean isSurrogate(int c) {
        return ((c>=0xD800)&&(c<=0xDFFF));
    }
    public static boolean isLowSurrogate(int c) {
        return ((c>=0xDC00)&&(c<=0xDFFF));
    }
    public static boolean isHighSurrogate(int c) {
        return ((c>=0xD800)&&(c<=0xDBFF));
    }
}

---