JezK

Edit File: chardistribution.rb

######################## BEGIN LICENSE BLOCK ########################
# The Original Code is Mozilla Communicator client code.
# 
# The Initial Developer of the Original Code is
# Netscape Communications Corporation.
# Portions created by the Initial Developer are Copyright (C) 1998
# the Initial Developer. All Rights Reserved.
# 
# Contributor(s):
#   Jeff Hodges
#   Mark Pilgrim - port to Python
#
# This library is free software; you can redistribute it and/or
# modify it under the terms of the GNU Lesser General Public
# License as published by the Free Software Foundation; either
# version 2.1 of the License, or (at your option) any later version.
# 
# This library is distributed in the hope that it will be useful,
# but WITHOUT ANY WARRANTY; without even the implied warranty of
# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the GNU
# Lesser General Public License for more details.
# 
# You should have received a copy of the GNU Lesser General Public
# License along with this library; if not, write to the Free Software
# Foundation, Inc., 51 Franklin St, Fifth Floor, Boston, MA
# 02110-1301  USA
######################### END LICENSE BLOCK #########################

module CharDet
  ENOUGH_DATA_THRESHOLD = 1024
  SURE_YES = 0.99
  SURE_NO = 0.01

class CharDistributionAnalysis
    def initialize
      @charToFreqOrder = nil # Mapping table to get frequency order from char order (get from GetOrder())
      @tableSize = nil # Size of above table
      @typicalDistributionRatio = nil # This is a constant value which varies from language to language, used in calculating confidence.  See http://www.mozilla.org/projects/intl/UniversalCharsetDetection.html for further detail.
      reset()
    end

def reset
      # # """reset analyser, clear any state"""
      @done = false # If this flag is set to constants.True, detection is done and conclusion has been made
      @totalChars = 0 # Total characters encountered
      @freqChars = 0 # The number of characters whose frequency order is less than 512
    end

def feed(aStr, aCharLen)
      # # """feed a character with known length"""
      if aCharLen == 2
        # we only care about 2-bytes character in our distribution analysis
        order = get_order(aStr)
      else
        order = -1
      end
      if order >= 0
        @totalChars += 1
        # order is valid
        if order < @tableSize
          if 512 > @charToFreqOrder[order]
            @freqChars += 1
          end
        end
      end
    end

def get_confidence
      # """return confidence based on existing data"""
      # if we didn't receive any character in our consideration range, return negative answer
      if @totalChars <= 0
        return SURE_NO
      end

if @totalChars != @freqChars
        r = @freqChars / ((@totalChars - @freqChars) * @typicalDistributionRatio)
        if r < SURE_YES
          return r
        end
      end

# normalize confidence (we don't want to be 100% sure)
      return SURE_YES
    end

def got_enough_data
      # It is not necessary to receive all data to draw conclusion. For charset detection,
      # certain amount of data is enough
      return @totalChars > ENOUGH_DATA_THRESHOLD
    end

def get_order(aStr)
      # We do not handle characters based on the original encoding string, but 
      # convert this encoding string to a number, here called order.
      # This allows multiple encodings of a language to share one frequency table.
      return -1
    end
  end

class EUCTWDistributionAnalysis < CharDistributionAnalysis
    def initialize
      super()
      @charToFreqOrder = EUCTWCharToFreqOrder
      @tableSize = EUCTW_TABLE_SIZE
      @typicalDistributionRatio = EUCTW_TYPICAL_DISTRIBUTION_RATIO
    end

def get_order(aStr)
      # for euc-TW encoding, we are interested 
      #   first  byte range: 0xc4 -- 0xfe
      #   second byte range: 0xa1 -- 0xfe
      # no validation needed here. State machine has done that
      if aStr[0, 1] >= "\xC4"
        bytes = aStr.bytes.to_a
        return 94 * (bytes[0] - 0xC4) + bytes[1] - 0xA1
      else
        return -1
      end
    end

def get_confidence
      if @freqChars <= MINIMUM_DATA_THRESHOLD
        return SURE_NO
      end

super
    end
  end

class EUCKRDistributionAnalysis < CharDistributionAnalysis
    def initialize
      super()
      @charToFreqOrder = EUCKRCharToFreqOrder
      @tableSize = EUCKR_TABLE_SIZE
      @typicalDistributionRatio = EUCKR_TYPICAL_DISTRIBUTION_RATIO
    end

def get_order(aStr)
      # for euc-KR encoding, we are interested 
      #   first  byte range: 0xb0 -- 0xfe
      #   second byte range: 0xa1 -- 0xfe
      # no validation needed here. State machine has done that
      if aStr[0, 1] >= "\xB0"
        bytes = aStr.bytes.to_a
        return 94 * (bytes[0] - 0xB0) + bytes[1] - 0xA1
      else
        return -1
      end
    end
  end

class GB18030DistributionAnalysis < CharDistributionAnalysis
    def initialize
      super()
      @charToFreqOrder = GB18030CharToFreqOrder
      @tableSize = GB18030_TABLE_SIZE
      @typicalDistributionRatio = GB18030_TYPICAL_DISTRIBUTION_RATIO
    end

def get_order(aStr)
      # for GB18030 encoding, we are interested 
      #  first  byte range: 0xb0 -- 0xfe
      #  second byte range: 0xa1 -- 0xfe
      # no validation needed here. State machine has done that
      if (aStr[0, 1] >= "\xB0") and (aStr[1, 1] >= "\xA1")
        bytes = aStr.bytes.to_a
        return 94 * (bytes[0] - 0xB0) + bytes[1] - 0xA1
      else
        return -1
      end
    end
  end

class Big5DistributionAnalysis < CharDistributionAnalysis
    def initialize
      super
      @charToFreqOrder = Big5CharToFreqOrder
      @tableSize = BIG5_TABLE_SIZE
      @typicalDistributionRatio = BIG5_TYPICAL_DISTRIBUTION_RATIO
    end

def get_order(aStr)
      # for big5 encoding, we are interested 
      #   first  byte range: 0xa4 -- 0xfe
      #   second byte range: 0x40 -- 0x7e , 0xa1 -- 0xfe
      # no validation needed here. State machine has done that
      if aStr[0, 1] >= "\xA4"
        bytes = aStr.bytes.to_a
        if aStr[1, 1] >= "\xA1"
          return 157 * (bytes[0] - 0xA4) + bytes[1] - 0xA1 + 63
        else
          return 157 * (bytes[0] - 0xA4) + bytes[1] - 0x40
        end
      else
        return -1
      end
    end
  end

class SJISDistributionAnalysis < CharDistributionAnalysis
    def initialize
      super()
      @charToFreqOrder = JISCharToFreqOrder
      @tableSize = JIS_TABLE_SIZE
      @typicalDistributionRatio = JIS_TYPICAL_DISTRIBUTION_RATIO
    end

def get_order(aStr)
      # for sjis encoding, we are interested 
      #   first  byte range: 0x81 -- 0x9f , 0xe0 -- 0xfe
      #   second byte range: 0x40 -- 0x7e,  0x81 -- oxfe
      # no validation needed here. State machine has done that
      bytes = aStr.bytes.to_a
      if (aStr[0, 1] >= "\x81") and (aStr[0, 1] <= "\x9F")
        order = 188 * (bytes[0] - 0x81)
      elsif (aStr[0, 1] >= "\xE0") and (aStr[0, 1] <= "\xEF")
        order = 188 * (bytes[0] - 0xE0 + 31)
      else
        return -1
      end
      order = order + bytes[1] - 0x40
      if aStr[1, 1] > "\x7F"
        order =- 1
      end
      return order
    end
  end

class EUCJPDistributionAnalysis < CharDistributionAnalysis
    def initialize
      super()
      @charToFreqOrder = JISCharToFreqOrder
      @tableSize = JIS_TABLE_SIZE
      @typicalDistributionRatio = JIS_TYPICAL_DISTRIBUTION_RATIO
    end

def get_order(aStr)
      # for euc-JP encoding, we are interested 
      #   first  byte range: 0xa0 -- 0xfe
      #   second byte range: 0xa1 -- 0xfe
      # no validation needed here. State machine has done that
      if aStr[0, 1] >= "\xA0"
        bytes = aStr.bytes.to_a
        return 94 * (bytes[0] - 0xA1) + bytes[1] - 0xa1
      else
        return -1
      end
    end
  end
end

JezK

Edit File: chardistribution.rb

FAIRNETZT