Add support for Unicode surrogate characters and planes above zero.HEADmaster

If the selected font supports the required characters, the text will be displayed correctly.
If the selected font does not support such characters, empty boxes will be displayed in place of the expected text.

Part of the code changes comes from Qt4 code.

Signed-off-by: Michele Calgaro <michele.calgaro@yahoo.it>

7 files changed, 72 insertions, 111 deletions

diff --git a/src/kernel/tqfontengine_p.h b/src/kernel/tqfontengine_p.h
index dcd306fc0..fe8714ef3 100644
--- a/src/kernel/tqfontengine_p.h
+++ b/src/kernel/tqfontengine_p.h
@@ -48,16 +48,18 @@
 
 #include "tqtextengine_p.h"
 
-class TQPaintDevice;
+typedef unsigned int glyph_t;
+typedef int advance_t;
 
 struct glyph_metrics_t;
-class TQChar;
-typedef unsigned short glyph_t;
 struct qoffset_t;
-typedef int advance_t;
-class TQOpenType;
 struct TransformedFont;
 
+class TQChar;
+class TQOpenType;
+class TQPaintDevice;
+
+
 #if defined( TQ_WS_X11 ) || defined( TQ_WS_WIN) || defined( TQ_WS_MAC )
 class TQFontEngine : public TQShared
 {
diff --git a/src/kernel/tqfontengine_x11.cpp b/src/kernel/tqfontengine_x11.cpp
index f39304c21..b3461a6ff 100644
--- a/src/kernel/tqfontengine_x11.cpp
+++ b/src/kernel/tqfontengine_x11.cpp
@@ -488,7 +488,15 @@ TQFontEngine::Error TQFontEngineXLFD::stringToCMap( const TQChar *str, int len,
 		chars[i] = (str[i].unicode() == 0xa0 ? 0x20 :
 			    (mirrored ? ::mirroredChar(str[i]).unicode() : str[i].unicode()));
 	}
-	_codec->fromUnicodeInternal( chars, glyphs, len );
+	// XLFD does not support unicode characters above 0xFFFF, so casting to ushort
+	// does not cause real loss
+	ushort *us_glyphs = new ushort[len];
+	_codec->fromUnicodeInternal( chars, us_glyphs, len );
+	for ( int i = 0; i < len; ++i ) {
+	    glyphs[i] = us_glyphs[i];
+	}
+	delete[] us_glyphs;
+
 	if (chars != str)
 	    free( chars );
     } else {
@@ -1523,6 +1531,18 @@ static glyph_t getAdobeCharIndex(XftFont *font, int cmap, uint ucs4)
     return g;
 }
 
+static uint getChar(const TQChar *str, int &i, const int len)
+{
+    uint uc = str[i].unicode();
+    if (uc >= 0xd800 && uc < 0xdc00 && i < len-1) {
+	uint low = str[++i].unicode();
+	if (low >= 0xdc00 && low < 0xe000) {
+	    uc = (uc - 0xd800)*0x400 + (low - 0xdc00) + 0x10000;
+	}
+    }
+    return uc;
+}
+
 TQFontEngine::Error TQFontEngineXft::stringToCMap( const TQChar *str, int len, glyph_t *glyphs, advance_t *advances, int *nglyphs, bool mirrored ) const
 {
     if ( *nglyphs < len ) {
@@ -1530,52 +1550,35 @@ TQFontEngine::Error TQFontEngineXft::stringToCMap( const TQChar *str, int len, g
 	return OutOfMemory;
     }
 
-    if (_cmap != -1) {
-	for ( int i = 0; i < len; ++i ) {
-	    unsigned short uc = str[i].unicode();
-            if (mirrored)
-                uc = ::mirroredChar(str[i]).unicode();
-	    glyphs[i] = uc < cmapCacheSize ? cmapCache[uc] : 0;
-	    if ( !glyphs[i] ) {
-		glyph_t glyph = XftCharIndex(0, _font, uc);
-                if (!glyph)
-                    glyph = getAdobeCharIndex(_font, _cmap, uc);
-		glyphs[i] = glyph;
-		if ( uc < cmapCacheSize )
-		    ((TQFontEngineXft *)this)->cmapCache[uc] = glyph;
+    int glyph_pos = 0;
+    for ( int i = 0; i < len; ++i ) {
+	uint uc = getChar(str, i, len);
+	if ( uc == 0xa0 )
+	    uc = 0x20;
+	if ( mirrored )
+	    uc = ::mirroredChar(uc).unicode();
+	glyphs[glyph_pos] = uc < cmapCacheSize ? cmapCache[uc] : 0;
+	if ( !glyphs[glyph_pos] ) {
+	    glyph_t glyph = 0;
+	    if (XftCharExists(0, _font, uc)) {
+		glyph = XftCharIndex(0, _font, uc);
 	    }
-	}
-    } else if ( mirrored ) {
-	for ( int i = 0; i < len; ++i ) {
-	    unsigned short uc = ::mirroredChar(str[i]).unicode();
-	    glyphs[i] = uc < cmapCacheSize ? cmapCache[uc] : 0;
-	    if ( !glyphs[i] ) {
-		if (uc == 0xa0)
-		    uc = 0x20;
-		glyph_t glyph = XftCharIndex(0, _font, uc);
-		glyphs[i] = glyph;
-		if ( uc < cmapCacheSize )
-		    ((TQFontEngineXft *)this)->cmapCache[uc] = glyph;
+	    if ( !glyph && _cmap != -1 ) {
+		glyph = getAdobeCharIndex(_font, _cmap, uc);
 	    }
-	}
-    } else {
-	for ( int i = 0; i < len; ++i ) {
-	    unsigned short uc = str[i].unicode();
-	    glyphs[i] = uc < cmapCacheSize ? cmapCache[uc] : 0;
-	    if ( !glyphs[i] ) {
-		if (uc == 0xa0)
-		    uc = 0x20;
-		glyph_t glyph = XftCharIndex(0, _font, uc);
-		glyphs[i] = glyph;
-		if ( uc < cmapCacheSize )
+	    if ( glyph ) {
+		glyphs[glyph_pos] = glyph;
+		if ( uc < cmapCacheSize ) {
 		    ((TQFontEngineXft *)this)->cmapCache[uc] = glyph;
+		}
 	    }
 	}
+	++glyph_pos;
     }
 
     if ( advances ) {
-	for ( int i = 0; i < len; i++ ) {
-	    FT_UInt glyph = *(glyphs + i);
+	for ( int i = 0; i < glyph_pos; i++ ) {
+	    glyph_t glyph = *(glyphs + i);
 	    advances[i] = (glyph < widthCacheSize) ? widthCache[glyph] : 0;
 	    if ( !advances[i] ) {
 		XGlyphInfo gi;
@@ -1591,7 +1594,7 @@ TQFontEngine::Error TQFontEngineXft::stringToCMap( const TQChar *str, int len, g
 	}
     }
 
-    *nglyphs = len;
+    *nglyphs = glyph_pos;
     return NoError;
 }
 
diff --git a/src/kernel/tqpainter.cpp b/src/kernel/tqpainter.cpp
index 452af0a57..7b30e0af6 100644
--- a/src/kernel/tqpainter.cpp
+++ b/src/kernel/tqpainter.cpp
@@ -2883,12 +2883,12 @@ void qt_format_text( const TQFont& font, const TQRect &_r,
 	int desc = fm.descent();
 	height = -leading;
 
-	//tqDebug("\n\nbeginLayout: lw = %d, rectwidth=%d", lineWidth , r.width());
+	//tqDebug("beginLayout: lw = %d, rectwidth=%d", lineWidth , r.width());
 	while ( !textLayout.atEnd() ) {
 	    height += leading;
-	    textLayout.beginLine( lineWidth == INT_MAX ? lineWidth : lineWidth );
+	    textLayout.beginLine( lineWidth );
 	    //tqDebug("-----beginLine( %d )-----",  lineWidth );
-	    bool linesep = FALSE;
+	    bool linesep = false;
 	    while ( 1 ) {
 		TQTextItem ti = textLayout.currentItem();
  		//tqDebug("item: from=%d, ch=%x", ti.from(), text.unicode()[ti.from()].unicode() );
@@ -2913,7 +2913,7 @@ void qt_format_text( const TQFont& font, const TQRect &_r,
 			ti.setWidth( tw );
 		}
 		if ( ti.isObject() && text.unicode()[ti.from()] == TQChar_linesep )
-		    linesep = TRUE;
+		    linesep = true;
 
 		if ( linesep || textLayout.addCurrentItem() != TQTextLayout::Ok || textLayout.atEnd() )
 		    break;
diff --git a/src/kernel/tqscriptengine.cpp b/src/kernel/tqscriptengine.cpp
index 2d97c84d5..588596716 100644
--- a/src/kernel/tqscriptengine.cpp
+++ b/src/kernel/tqscriptengine.cpp
@@ -279,15 +279,13 @@ void qt_heuristicPosition(TQShaperItem *item)
 
 
 
-// set the glyph attributes heuristically. Assumes a 1 to 1 relationship between chars and glyphs
-// and no reordering.
-// also computes logClusters heuristically
+// Set the glyph attributes heuristically. Assumes no reordering.
+// Also computes logClusters heuristically
 static void heuristicSetGlyphAttributes(TQShaperItem *item, const TQChar *uc, int length)
 {
     // justification is missing here!!!!!
 
-    if ( item->num_glyphs != length )
-       tqWarning("TQScriptEngine::heuristicSetGlyphAttributes: char length and num glyphs disagree" );
+    Q_ASSERT(item->num_glyphs <= length);
 
     unsigned short *logClusters = item->log_clusters;
 
diff --git a/src/kernel/tqtextengine.cpp b/src/kernel/tqtextengine.cpp
index 6c046a5b8..f50d849cc 100644
--- a/src/kernel/tqtextengine.cpp
+++ b/src/kernel/tqtextengine.cpp
@@ -150,7 +150,7 @@ static TQChar::Direction basicDirection( const TQString &str )
 }
 
 
-static void tqAppendItems(TQTextEngine *engine, int &start, int &stop, BidiControl &control, TQChar::Direction dir )
+static void appendItems(TQTextEngine *engine, int &start, int &stop, BidiControl &control, TQChar::Direction dir )
 {
     TQScriptItemArray &items = engine->items;
     const TQChar *text = engine->string.unicode();
@@ -243,9 +243,6 @@ static void tqAppendItems(TQTextEngine *engine, int &start, int &stop, BidiContr
     start = stop;
 }
 
-typedef void (* fAppendItems)(TQTextEngine *, int &start, int &stop, BidiControl &control, TQChar::Direction dir);
-static fAppendItems appendItems = tqAppendItems;
-
 // creates the next TQScript items.
 static void bidiItemize( TQTextEngine *engine, bool rightToLeft, int mode )
 {
@@ -666,8 +663,6 @@ static void bidiItemize( TQTextEngine *engine, bool rightToLeft, int mode )
 
     if ( sor <= eor )
 	appendItems(engine, sor, eor, control, dir);
-
-
 }
 
 void TQTextEngine::bidiReorder( int numItems, const TQ_UINT8 *levels, int *visualOrder )
diff --git a/src/kernel/tqtextengine_p.h b/src/kernel/tqtextengine_p.h
index d092a3e7d..d3e8ea3ee 100644
--- a/src/kernel/tqtextengine_p.h
+++ b/src/kernel/tqtextengine_p.h
@@ -86,57 +86,28 @@ struct glyph_metrics_t
     int yoff;
 };
 
-#if defined( TQ_WS_X11 )
-typedef unsigned short glyph_t;
-
-struct qoffset_t {
-    short x;
-    short y;
-};
-
+typedef unsigned int glyph_t;
 typedef int advance_t;
 
-struct TQScriptAnalysis
-{
-    unsigned short script    : 7;
-    unsigned short bidiLevel : 6;  // Unicode Bidi algorithm embedding level (0-61)
-    unsigned short override  : 1;  // Set when in LRO/RLO embedding
-    unsigned short reserved  : 2;
-    bool operator == ( const TQScriptAnalysis &other ) {
-	return
-	    script == other.script &&
-	    bidiLevel == other.bidiLevel;
-	// ###
-// 	    && override == other.override;
-    }
+#if defined( TQ_WS_X11 ) || defined( TQ_WS_MAC )
 
-};
-
-#elif defined( TQ_WS_MAC )
-
-typedef unsigned short glyph_t;
-
-struct qoffset_t {
+struct qoffset_t
+{
     short x;
     short y;
 };
 
-typedef int advance_t;
-
 struct TQScriptAnalysis
 {
     unsigned short script    : 7;
     unsigned short bidiLevel : 6;  // Unicode Bidi algorithm embedding level (0-61)
     unsigned short override  : 1;  // Set when in LRO/RLO embedding
     unsigned short reserved  : 2;
-    bool operator == ( const TQScriptAnalysis &other ) {
-	return
-	    script == other.script &&
-	    bidiLevel == other.bidiLevel;
-	// ###
-// 	    && override == other.override;
-    }
 
+    bool operator== ( const TQScriptAnalysis &other )
+    {
+	return script == other.script && bidiLevel == other.bidiLevel;
+    }
 };
 
 #elif defined( TQ_WS_WIN )
@@ -144,15 +115,12 @@ struct TQScriptAnalysis
 // do not change the definitions below unless you know what you are doing!
 // it is designed to be compatible with the types found in uniscribe.
 
-typedef unsigned short glyph_t;
-
-struct qoffset_t {
+struct qoffset_t
+{
     int x;
     int y;
 };
 
-typedef int advance_t;
-
 struct TQScriptAnalysis {
     unsigned short script         :10;
     unsigned short rtl            :1;
@@ -174,13 +142,9 @@ struct TQScriptAnalysis {
     unsigned short engineReserved    :2;
 };
 
-inline bool operator == ( const TQScriptAnalysis &sa1, const TQScriptAnalysis &sa2 )
+inline bool operator== ( const TQScriptAnalysis &sa1, const TQScriptAnalysis &sa2 )
 {
-    return
-	sa1.script == sa2.script &&
-	sa1.bidiLevel == sa2.bidiLevel;
-	// ###
-// 	    && override == other.override;
+    return sa1.script == sa2.script && sa1.bidiLevel == sa2.bidiLevel;
 }
 
 #endif
diff --git a/src/kernel/tqtextengine_unix.cpp b/src/kernel/tqtextengine_unix.cpp
index 0d5e3b920..0de67f04a 100644
--- a/src/kernel/tqtextengine_unix.cpp
+++ b/src/kernel/tqtextengine_unix.cpp
@@ -80,8 +80,8 @@ void TQTextEngine::shape( int item ) const
 	si.fontEngine = fnt->engineForScript( script );
     si.fontEngine->ref();
 
-	si.ascent = si.fontEngine->ascent();
-	si.descent = si.fontEngine->descent();
+    si.ascent = si.fontEngine->ascent();
+    si.descent = si.fontEngine->descent();
     si.num_glyphs = 0;
 
     if ( si.fontEngine && si.fontEngine != (TQFontEngine*)-1 ) {
@@ -96,7 +96,6 @@ void TQTextEngine::shape( int item ) const
         shaper_item.has_positioning = FALSE;
 
         while (1) {
-//         tqDebug("    . num_glyphs=%d, used=%d, item.num_glyphs=%d", num_glyphs, used, shaper_item.num_glyphs);
             ensureSpace(shaper_item.num_glyphs);
             shaper_item.num_glyphs = num_glyphs - used;
 //          tqDebug("    .. num_glyphs=%d, used=%d, item.num_glyphs=%d", num_glyphs, used, shaper_item.num_glyphs);