/**********************************************************************
 * File:        reject.cpp  (Formerly reject.c)
 * Description: Rejection functions used in tessedit
 * Author:		Phil Cheatle
 * Created:		Wed Sep 23 16:50:21 BST 1992
 *
 * (C) Copyright 1992, Hewlett-Packard Ltd.
 ** Licensed under the Apache License, Version 2.0 (the "License");
 ** you may not use this file except in compliance with the License.
 ** You may obtain a copy of the License at
 ** http://www.apache.org/licenses/LICENSE-2.0
 ** Unless required by applicable law or agreed to in writing, software
 ** distributed under the License is distributed on an "AS IS" BASIS,
 ** WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
 ** See the License for the specific language governing permissions and
 ** limitations under the License.
 *
 **********************************************************************/

#include "mfcpch.h"
#include          "tessvars.h"
#include          <assert.h>
#include          <errno.h>
#include          <ctype.h>
#include          <string.h>
#include          "memry.h"
#include          "reject.h"
#include          "tfacep.h"
#include          "mainblk.h"
#include          "charcut.h"
#include          "imgs.h"
#include          "scaleimg.h"
#include          "control.h"
#include          "docqual.h"
#include          "secname.h"
#include          "globals.h"

/* #define SECURE_NAMES done in secnames.h when necessary */


CLISTIZEH (STRING) CLISTIZE (STRING)
#define EXTERN
EXTERN
INT_VAR (tessedit_reject_mode, 0, "Rejection algorithm");
EXTERN
INT_VAR (tessedit_ok_mode, 5, "Acceptance decision algorithm");
EXTERN
BOOL_VAR (tessedit_rejection_debug, FALSE, "Adaption debug");
EXTERN
BOOL_VAR (tessedit_rejection_stats, FALSE, "Show NN stats");
EXTERN
BOOL_VAR (tessedit_flip_0O, TRUE, "Contextual 0O O0 flips");
EXTERN
double_VAR (tessedit_lower_flip_hyphen, 1.5,
"Aspect ratio dot/hyphen test");
EXTERN
double_VAR (tessedit_upper_flip_hyphen, 1.8,
"Aspect ratio dot/hyphen test");

EXTERN
BOOL_VAR (rej_trust_doc_dawg, FALSE,
"Use DOC dawg in 11l conf. detector");
EXTERN
BOOL_VAR (rej_1Il_use_dict_word, FALSE, "Use dictword test");
EXTERN
BOOL_VAR (rej_1Il_trust_permuter_type, TRUE, "Dont double check");

EXTERN
BOOL_VAR (one_ell_conflict_default, TRUE, "one_ell_conflict default");
EXTERN
BOOL_VAR (show_char_clipping, FALSE, "Show clip image window?");
EXTERN
BOOL_VAR (nn_debug, FALSE, "NN DEBUGGING?");
EXTERN
BOOL_VAR (nn_reject_debug, FALSE, "NN DEBUG each char?");
EXTERN
BOOL_VAR (nn_lax, FALSE, "Use 2nd rate matches");
EXTERN
BOOL_VAR (nn_double_check_dict, FALSE, "Double check");
EXTERN
BOOL_VAR (nn_conf_double_check_dict, TRUE,
"Double check for confusions");
EXTERN
BOOL_VAR (nn_conf_1Il, TRUE, "NN use 1Il conflicts");
EXTERN
BOOL_VAR (nn_conf_Ss, TRUE, "NN use Ss conflicts");
EXTERN
BOOL_VAR (nn_conf_hyphen, TRUE, "NN hyphen conflicts");
EXTERN
BOOL_VAR (nn_conf_test_good_qual, FALSE, "NN dodgy 1Il cross check");
EXTERN
BOOL_VAR (nn_conf_test_dict, TRUE, "NN dodgy 1Il cross check");
EXTERN
BOOL_VAR (nn_conf_test_sensible, TRUE, "NN dodgy 1Il cross check");
EXTERN
BOOL_VAR (nn_conf_strict_on_dodgy_chs, TRUE,
"Require stronger NN match");
EXTERN
double_VAR (nn_dodgy_char_threshold, 0.99, "min accept score");
EXTERN
INT_VAR (nn_conf_accept_level, 4, "NN accept dodgy 1Il matches? ");
EXTERN
INT_VAR (nn_conf_initial_i_level, 3,
"NN accept initial Ii match level ");

EXTERN
BOOL_VAR (no_unrej_dubious_chars, TRUE, "Dubious chars next to reject?");
EXTERN
BOOL_VAR (no_unrej_no_alphanum_wds, TRUE, "Stop unrej of non A/N wds?");
EXTERN
BOOL_VAR (no_unrej_1Il, FALSE, "Stop unrej of 1Ilchars?");
EXTERN
BOOL_VAR (rej_use_tess_accepted, TRUE, "Individual rejection control");
EXTERN
BOOL_VAR (rej_use_tess_blanks, TRUE, "Individual rejection control");
EXTERN
BOOL_VAR (rej_use_good_perm, TRUE, "Individual rejection control");
EXTERN
BOOL_VAR (rej_use_sensible_wd, FALSE, "Extend permuter check");
EXTERN
BOOL_VAR (rej_alphas_in_number_perm, FALSE, "Extend permuter check");

EXTERN
double_VAR (rej_whole_of_mostly_reject_word_fract, 0.85,
"if >this fract");
EXTERN
INT_VAR (rej_mostly_reject_mode, 1,
"0-never, 1-afterNN, 2-after new xht");
EXTERN
double_VAR (tessed_fullstop_aspect_ratio, 1.2,
"if >this fract then reject");

EXTERN
INT_VAR (net_image_width, 40, "NN input image width");
EXTERN
INT_VAR (net_image_height, 36, "NN input image height");
EXTERN
INT_VAR (net_image_x_height, 22, "NN input image x_height");
EXTERN
INT_VAR (tessedit_image_border, 2, "Rej blbs near image edge limit");

/*
  Net input is assumed to have (net_image_width * net_image_height) input
  units of image pixels, followed by 0, 1, or N units representing the
  baseline position. 0 implies no baseline information. 1 implies a floating
  point value. N implies a "guage" of N units. For any char an initial set
  of these are ON, the remainder OFF to indicate the "level" of the
  baseline.

  HOWEVER!!!  NOTE THAT EACH NEW INPUT LAYER FORMAT EXPECTS TO BE RUN WITH A
  DIFFERENT tessed/netmatch/nmatch.c MODULE. - These are classic C modules
  generated by aspirin with HARD CODED CONSTANTS
*/

EXTERN
INT_VAR (net_bl_nodes, 20, "Number of baseline nodes");

EXTERN
double_VAR (nn_reject_threshold, 0.5, "NN min accept score");
EXTERN
double_VAR (nn_reject_head_and_shoulders, 0.6, "top scores sep factor");

/* NOTE - ctoh doesn't handle "=" properly, hence \075 */
EXTERN
STRING_VAR (ok_single_ch_non_alphanum_wds, "-?\075",
"Allow NN to unrej");
EXTERN
STRING_VAR (ok_repeated_ch_non_alphanum_wds, "-?*\075",
"Allow NN to unrej");
EXTERN
STRING_VAR (conflict_set_I_l_1, "Il1[]", "Il1 conflict set");
EXTERN
STRING_VAR (conflict_set_S_s, "Ss$", "Ss conflict set");
EXTERN
STRING_VAR (conflict_set_hyphen, "-_~", "hyphen conflict set");
EXTERN
STRING_VAR (dubious_chars_left_of_reject, "!'+`()-./\\<>;:^_,~\"",
"Unreliable chars");
EXTERN
STRING_VAR (dubious_chars_right_of_reject, "!'+`()-./\\<>;:^_,~\"",
"Unreliable chars");

EXTERN
INT_VAR (min_sane_x_ht_pixels, 8, "Reject any x-ht lt or eq than this");

/*************************************************************************
 * set_done()
 *
 * Set the done flag based on the word acceptability criteria
 *************************************************************************/

void set_done(  //set done flag
              WERD_RES *word,
              inT16 pass) {
  /*
  0: Original heuristic used in Tesseract and Ray's prototype Resaljet
  */
  if (tessedit_ok_mode == 0) {
    /* NOTE - done even if word contains some or all spaces !!! */
    word->done = word->tess_accepted;
  }
  /*
  1: Reject words containing blanks and on pass 1 reject I/l/1 conflicts
  */
  else if (tessedit_ok_mode == 1) {
    word->done = word->tess_accepted &&
      (strchr (word->best_choice->string ().string (), ' ') == NULL);

    if (word->done && (pass == 1) && one_ell_conflict (word, FALSE))
      word->done = FALSE;
  }
  /*
  2: as 1 + only accept dict words or numerics in pass 1
  */
  else if (tessedit_ok_mode == 2) {
    word->done = word->tess_accepted &&
      (strchr (word->best_choice->string ().string (), ' ') == NULL);

    if (word->done && (pass == 1) && one_ell_conflict (word, FALSE))
      word->done = FALSE;

    if (word->done &&
      (pass == 1) &&
      (word->best_choice->permuter () != SYSTEM_DAWG_PERM) &&
      (word->best_choice->permuter () != FREQ_DAWG_PERM) &&
      (word->best_choice->permuter () != USER_DAWG_PERM) &&
    (word->best_choice->permuter () != NUMBER_PERM)) {
      #ifndef SECURE_NAMES
      if (tessedit_rejection_debug)
        tprintf ("\nVETO Tess accepting poor word \"%s\"\n",
          word->best_choice->string ().string ());
      #endif
      word->done = FALSE;
    }
  }
  /*
  3: as 2 + only accept dict words or numerics in pass 2 as well
  */
  else if (tessedit_ok_mode == 3) {
    word->done = word->tess_accepted &&
      (strchr (word->best_choice->string ().string (), ' ') == NULL);

    if (word->done && (pass == 1) && one_ell_conflict (word, FALSE))
      word->done = FALSE;

    if (word->done &&
      (word->best_choice->permuter () != SYSTEM_DAWG_PERM) &&
      (word->best_choice->permuter () != FREQ_DAWG_PERM) &&
      (word->best_choice->permuter () != USER_DAWG_PERM) &&
    (word->best_choice->permuter () != NUMBER_PERM)) {
      #ifndef SECURE_NAMES
      if (tessedit_rejection_debug)
        tprintf ("\nVETO Tess accepting poor word \"%s\"\n",
          word->best_choice->string ().string ());
      #endif
      word->done = FALSE;
    }
  }
  /*
  4: as 2 + reject dict ambigs in pass 1
  */
  else if (tessedit_ok_mode == 4) {
    word->done = word->tess_accepted &&
      (strchr (word->best_choice->string ().string (), ' ') == NULL);

    if (word->done && (pass == 1) && one_ell_conflict (word, FALSE))
      word->done = FALSE;

    if (word->done &&
      (pass == 1) &&
      (((word->best_choice->permuter () != SYSTEM_DAWG_PERM) &&
      (word->best_choice->permuter () != FREQ_DAWG_PERM) &&
      (word->best_choice->permuter () != USER_DAWG_PERM) &&
      (word->best_choice->permuter () != NUMBER_PERM)) ||
    (test_ambig_word (word)))) {
      #ifndef SECURE_NAMES
      if (tessedit_rejection_debug)
        tprintf ("\nVETO Tess accepting poor word \"%s\"\n",
          word->best_choice->string ().string ());
      #endif
      word->done = FALSE;
    }
  }
  /*
  5: as 3 + reject dict ambigs in both passes
  */
  else if (tessedit_ok_mode == 5) {
    word->done = word->tess_accepted &&
      (strchr (word->best_choice->string ().string (), ' ') == NULL);

    if (word->done && (pass == 1) && one_ell_conflict (word, FALSE))
      word->done = FALSE;

    if (word->done &&
      (((word->best_choice->permuter () != SYSTEM_DAWG_PERM) &&
      (word->best_choice->permuter () != FREQ_DAWG_PERM) &&
      (word->best_choice->permuter () != USER_DAWG_PERM) &&
      (word->best_choice->permuter () != NUMBER_PERM)) ||
    (test_ambig_word (word)))) {
      #ifndef SECURE_NAMES
      if (tessedit_rejection_debug)
        tprintf ("\nVETO Tess accepting poor word \"%s\"\n",
          word->best_choice->string ().string ());
      #endif
      word->done = FALSE;
    }
  }

  else {
    tprintf ("BAD tessedit_ok_mode\n");
    err_exit();
  }
}


/*************************************************************************
 * make_reject_map()
 *
 * Sets the done flag to indicate whether the resylt is acceptable.
 *
 * Sets a reject map for the word.
 *************************************************************************/

void make_reject_map(            //make rej map for wd //detailed results
                     WERD_RES *word,
                     BLOB_CHOICE_LIST_CLIST *blob_choices,
                     ROW *row,
                     inT16 pass  //1st or 2nd?
                    ) {
  int i;
  int offset;

  flip_0O(word);
  check_debug_pt (word, -1);     //For trap only
  set_done(word, pass);  //Set acceptance
  word->reject_map.initialise (word->best_choice->lengths ().length ());
  reject_blanks(word);
  /*
  0: Rays original heuristic - the baseline
  */
  if (tessedit_reject_mode == 0) {
    if (!word->done)
      reject_poor_matches(word, blob_choices);
  }
  /*
  5: Reject I/1/l from words where there is no strong contextual confirmation;
    the whole of any unacceptable words (incl PERM rej of dubious 1/I/ls);
    and the whole of any words which are very small
  */
  else if (tessedit_reject_mode == 5) {
    if (bln_x_height / word->denorm.scale () <= min_sane_x_ht_pixels)
      word->reject_map.rej_word_small_xht ();
    else {
      one_ell_conflict(word, TRUE);
      /*
        Originally the code here just used the done flag. Now I have duplicated
        and unpacked the conditions for setting the done flag so that each
        mechanism can be turned on or off independently. This works WITHOUT
        affecting the done flag setting.
      */
      if (rej_use_tess_accepted && !word->tess_accepted)
        word->reject_map.rej_word_not_tess_accepted ();

      if (rej_use_tess_blanks &&
        (strchr (word->best_choice->string ().string (), ' ') != NULL))
        word->reject_map.rej_word_contains_blanks ();

      if (rej_use_good_perm) {
        if (((word->best_choice->permuter () == SYSTEM_DAWG_PERM) ||
          (word->best_choice->permuter () == FREQ_DAWG_PERM) ||
          (word->best_choice->permuter () == USER_DAWG_PERM)) &&
          (!rej_use_sensible_wd ||
          (acceptable_word_string
          (word->best_choice->string ().string (),
           word->best_choice->lengths ().string ()) !=
        AC_UNACCEPTABLE))) {
          //PASSED TEST
        }
        else if (word->best_choice->permuter () == NUMBER_PERM) {
          if (rej_alphas_in_number_perm) {
            for (i = 0, offset = 0;
                 word->best_choice->string ()[offset] != '\0';
                 offset += word->best_choice->lengths()[i++]) {
              if (word->reject_map[i].accepted () &&
                  unicharset.get_isalpha (word->best_choice->string ().string()
                                          + offset,
                                          word->best_choice->lengths()[i]))
                word->reject_map[i].setrej_bad_permuter ();
              //rej alpha
            }
          }
        }
        else {
          word->reject_map.rej_word_bad_permuter ();
        }
      }

      /* Ambig word rejection was here once !!*/

    }
  }
  else {
    tprintf ("BAD tessedit_reject_mode\n");
    err_exit();
  }

  if (tessedit_image_border > -1)
    reject_edge_blobs(word);

  check_debug_pt (word, 10);
  if (tessedit_rejection_debug) {
    tprintf ("Permuter Type = %d\n", word->best_choice->permuter ());
    tprintf ("Certainty: %f     Rating: %f\n",
      word->best_choice->certainty (), word->best_choice->rating ());
    tprintf ("Dict word: %d\n",
      dict_word (word->best_choice->string ().string ()));
  }

  /* Un-reject any rejected characters if NN permits */

  flip_hyphens(word);
  check_debug_pt (word, 20);
}


void reject_blanks(WERD_RES *word) {
  inT16 i;
  inT16 offset;

  for (i = 0, offset = 0; word->best_choice->string ()[offset] != '\0';
       offset += word->best_choice->lengths ()[i], i += 1) {
    if (word->best_choice->string ()[offset] == ' ')
                                 //rej unrecognised blobs
      word->reject_map[i].setrej_tess_failure ();
  }
}


void reject_I_1_L(WERD_RES *word) {
  inT16 i;
  inT16 offset;

  for (i = 0, offset = 0; word->best_choice->string ()[offset] != '\0';
       offset += word->best_choice->lengths ()[i], i += 1) {
    if (STRING (conflict_set_I_l_1).
    contains (word->best_choice->string ()[offset])) {
                                 //rej 1Il conflict
      word->reject_map[i].setrej_1Il_conflict ();
    }
  }
}


void reject_poor_matches(  //detailed results
                         WERD_RES *word,
                         BLOB_CHOICE_LIST_CLIST *blob_choices) {
  float threshold;
  inT16 i = 0;
  inT16 offset = 0;
                                 //super iterator
  BLOB_CHOICE_LIST_C_IT list_it = blob_choices;
  BLOB_CHOICE_IT choice_it;      //real iterator

  #ifndef SECURE_NAMES
  if (strlen (word->best_choice->lengths ().string ()) != list_it.length ()) {
    tprintf
      ("ASSERT FAIL string:\"%s\"; strlen=%d; choices len=%d; blob len=%d\n",
      word->best_choice->string ().string (),
      strlen (word->best_choice->lengths ().string ()), list_it.length (),
      word->outword->blob_list ()->length ());
  }
  #endif
  ASSERT_HOST (strlen (word->best_choice->lengths ().string ()) ==
    list_it.length ());
  ASSERT_HOST (word->outword->blob_list ()->length () == list_it.length ());
  threshold = compute_reject_threshold (blob_choices);

  for (list_it.mark_cycle_pt ();
  !list_it.cycled_list (); list_it.forward (), i++,
           offset += word->best_choice->lengths ()[i]) {
    /* NB - only compares the threshold against the TOP choice char in the
      choices list for a blob !! - the selected one may be below the threshold */
    choice_it.set_to_list (list_it.data ());
    if ((word->best_choice->string ()[offset] == ' ') ||
      (choice_it.length () == 0))
                                 //rej unrecognised blobs
      word->reject_map[i].setrej_tess_failure ();
    else if (choice_it.data ()->certainty () < threshold)
                                 //rej poor score blob
      word->reject_map[i].setrej_poor_match ();
  }
}


/**********************************************************************
 * compute_reject_threshold
 *
 * Set a rejection threshold for this word.
 * Initially this is a trivial function which looks for the largest
 * gap in the certainty value.
 **********************************************************************/

float compute_reject_threshold(  //compute threshold //detailed results
                               BLOB_CHOICE_LIST_CLIST *blob_choices) {
  inT16 index;                   //to ratings
  inT16 blob_count;              //no of blobs in word
  inT16 ok_blob_count = 0;       //non TESS rej blobs in word
  float *ratings;                //array of confidences
  float threshold;               //rejection threshold
  float bestgap;                 //biggest gap
  float gapstart;                //bottom of gap
                                 //super iterator
  BLOB_CHOICE_LIST_C_IT list_it = blob_choices;
  BLOB_CHOICE_IT choice_it;      //real iterator

  blob_count = blob_choices->length ();
  ratings = (float *) alloc_mem (blob_count * sizeof (float));
  for (list_it.mark_cycle_pt (), index = 0;
  !list_it.cycled_list (); list_it.forward (), index++) {
    choice_it.set_to_list (list_it.data ());
    if (choice_it.length () > 0) {
      ratings[ok_blob_count] = choice_it.data ()->certainty ();
      //get in an array
      //                 tprintf("Rating[%d]=%c %g %g\n",
      //                         index,choice_it.data()->char_class(),
      //                         choice_it.data()->rating(),choice_it.data()->certainty());
      ok_blob_count++;
    }
  }
  ASSERT_HOST (index == blob_count);
  qsort (ratings, ok_blob_count, sizeof (float), sort_floats);
  //sort them
  bestgap = 0;
  gapstart = ratings[0] - 1;     //all reject if none better
  if (ok_blob_count >= 3) {
    for (index = 0; index < ok_blob_count - 1; index++) {
      if (ratings[index + 1] - ratings[index] > bestgap) {
        bestgap = ratings[index + 1] - ratings[index];
        //find biggest
        gapstart = ratings[index];
      }
    }
  }
  threshold = gapstart + bestgap / 2;
  //      tprintf("First=%g, last=%g, gap=%g, threshold=%g\n",
  //              ratings[0],ratings[index],bestgap,threshold);

  free_mem(ratings);
  return threshold;
}


/**********************************************************************
 * sort_floats
 *
 * qsort function to sort 2 floats.
 **********************************************************************/

int sort_floats(                   //qsort function
                const void *arg1,  //ptrs to floats
                const void *arg2) {
  float diff;                    //difference

  diff = *((float *) arg1) - *((float *) arg2);
  if (diff > 0)
    return 1;
  else if (diff < 0)
    return -1;
  else
    return 0;
}


/*************************************************************************
 * reject_edge_blobs()
 *
 * If the word is perilously close to the edge of the image, reject those blobs
 * in the word which are too close to the edge as they could be clipped.
 *************************************************************************/

void reject_edge_blobs(WERD_RES *word) {
  TBOX word_box = word->word->bounding_box ();
  TBOX blob_box;
  PBLOB_IT blob_it = word->outword->blob_list ();
  //blobs
  int blobindex = 0;
  float centre;

  if ((word_box.left () < tessedit_image_border) ||
    (word_box.bottom () < tessedit_image_border) ||
    (word_box.right () + tessedit_image_border >
    page_image.get_xsize () - 1) ||
  (word_box.top () + tessedit_image_border > page_image.get_ysize () - 1)) {
    ASSERT_HOST (word->reject_map.length () == blob_it.length ());
    for (blobindex = 0, blob_it.mark_cycle_pt ();
    !blob_it.cycled_list (); blobindex++, blob_it.forward ()) {
      blob_box = blob_it.data ()->bounding_box ();
      centre = (blob_box.left () + blob_box.right ()) / 2.0;
      if ((word->denorm.x (blob_box.left ()) < tessedit_image_border) ||
        (word->denorm.y (blob_box.bottom (), centre) <
        tessedit_image_border) ||
        (word->denorm.x (blob_box.right ()) + tessedit_image_border >
        page_image.get_xsize () - 1) ||
        (word->denorm.y (blob_box.top (), centre)
      + tessedit_image_border > page_image.get_ysize () - 1)) {
        word->reject_map[blobindex].setrej_edge_char ();
        //close to edge
      }
    }
  }
}


/**********************************************************************
 * one_ell_conflict()
 *
 * Identify words where there is a potential I/l/1 error.
 * - A bundle of contextual heuristics!
 **********************************************************************/

BOOL8 one_ell_conflict(WERD_RES *word_res, BOOL8 update_map) {
  const char *word;
  const char *lengths;
  inT16 word_len;                //its length
  inT16 first_alphanum_index_;
  inT16 first_alphanum_offset_;
  inT16 i;
  inT16 offset;
  BOOL8 non_conflict_set_char;   //non conf set a/n?
  BOOL8 conflict = FALSE;
  BOOL8 allow_1s;
  ACCEPTABLE_WERD_TYPE word_type;
  BOOL8 dict_perm_type;
  BOOL8 dict_word_ok;
  int dict_word_type;

  word = word_res->best_choice->string ().string ();
  lengths = word_res->best_choice->lengths().string();
  word_len = strlen (lengths);
  /*
    If there are no occurrences of the conflict set characters then the word
    is OK.
  */
  if (strpbrk (word, conflict_set_I_l_1.string ()) == NULL)
    return FALSE;

  /*
    There is a conflict if there are NO other (confirmed) alphanumerics apart
    from those in the conflict set.
  */

  for (i = 0, offset = 0, non_conflict_set_char = FALSE;
       (i < word_len) && !non_conflict_set_char; offset += lengths[i++])
    non_conflict_set_char =
        (unicharset.get_isalpha(word + offset, lengths[i]) ||
         unicharset.get_isdigit(word + offset, lengths[i])) &&
        !STRING (conflict_set_I_l_1).contains (word[offset]);
  if (!non_conflict_set_char) {
    if (update_map)
      reject_I_1_L(word_res);
    return TRUE;
  }

  /*
    If the word is accepted by a dawg permuter, and the first alpha character
    is "I" or "l", check to see if the alternative is also a dawg word. If it
    is, then there is a potential error otherwise the word is ok.
  */

  dict_perm_type = (word_res->best_choice->permuter () == SYSTEM_DAWG_PERM) ||
    (word_res->best_choice->permuter () == USER_DAWG_PERM) ||
    (rej_trust_doc_dawg &&
    (word_res->best_choice->permuter () == DOC_DAWG_PERM)) ||
    (word_res->best_choice->permuter () == FREQ_DAWG_PERM);
  dict_word_type = dict_word (word);
  dict_word_ok = (dict_word_type > 0) &&
    (rej_trust_doc_dawg || (dict_word_type != DOC_DAWG_PERM));

  if ((rej_1Il_use_dict_word && dict_word_ok) ||
    (rej_1Il_trust_permuter_type && dict_perm_type) ||
  (dict_perm_type && dict_word_ok)) {
    first_alphanum_index_ = first_alphanum_index (word, lengths);
    first_alphanum_offset_ = first_alphanum_offset (word, lengths);
    if (lengths[first_alphanum_index_] == 1 &&
        word[first_alphanum_offset_] == 'I') {
      word_res->best_choice->string ()[first_alphanum_offset_] = 'l';
      if (safe_dict_word (word) > 0) {
        word_res->best_choice->string ()[first_alphanum_offset_] = 'I';
        if (update_map)
          word_res->reject_map[first_alphanum_index_].
            setrej_1Il_conflict();
        return TRUE;
      }
      else {
        word_res->best_choice->string ()[first_alphanum_offset_] = 'I';
        return FALSE;
      }
    }

    if (lengths[first_alphanum_index_] == 1 &&
        word[first_alphanum_offset_] == 'l') {
      word_res->best_choice->string ()[first_alphanum_offset_] = 'I';
      if (safe_dict_word (word) > 0) {
        word_res->best_choice->string ()[first_alphanum_offset_] = 'l';
        if (update_map)
          word_res->reject_map[first_alphanum_index_].
            setrej_1Il_conflict();
        return TRUE;
      }
      else {
        word_res->best_choice->string ()[first_alphanum_offset_] = 'l';
        return FALSE;
      }
    }
    return FALSE;
  }

  /*
    NEW 1Il code. The old code relied on permuter types too much. In fact,
    tess will use TOP_CHOICE permute for good things like "palette".
    In this code the string is examined independently to see if it looks like
    a well formed word.
  */

  /*
    REGARDLESS OF PERMUTER, see if flipping a leading I/l generates a
    dictionary word.
  */
  first_alphanum_index_ = first_alphanum_index (word, lengths);
  first_alphanum_offset_ = first_alphanum_offset (word, lengths);
  if (lengths[first_alphanum_index_] == 1 &&
      word[first_alphanum_offset_] == 'l') {
    word_res->best_choice->string ()[first_alphanum_offset_] = 'I';
    if (safe_dict_word (word) > 0)
      return FALSE;
    else
      word_res->best_choice->string ()[first_alphanum_offset_] = 'l';
  }
  else if (lengths[first_alphanum_index_] == 1 &&
           word[first_alphanum_offset_] == 'I') {
    word_res->best_choice->string ()[first_alphanum_offset_] = 'l';
    if (safe_dict_word (word) > 0)
      return FALSE;
    else
      word_res->best_choice->string ()[first_alphanum_offset_] = 'I';
  }
  /*
    For strings containing digits:
      If there are no alphas OR the numeric permuter liked the word,
        reject any non 1 conflict chs
      Else reject all conflict chs
  */
  if (word_contains_non_1_digit (word, lengths)) {
    allow_1s = (alpha_count (word, lengths) == 0) ||
      (word_res->best_choice->permuter () == NUMBER_PERM);

    inT16 offset;
    conflict = FALSE;
    for (i = 0, offset = 0; word[offset] != '\0';
         offset += word_res->best_choice->lengths ()[i++]) {
      if ((!allow_1s || (word[offset] != '1')) &&
      STRING (conflict_set_I_l_1).contains (word[offset])) {
        if (update_map)
          word_res->reject_map[i].setrej_1Il_conflict ();
        conflict = TRUE;
      }
    }
    return conflict;
  }
  /*
    For anything else. See if it conforms to an acceptable word type. If so,
    treat accordingly.
  */
  word_type = acceptable_word_string (word, lengths);
  if ((word_type == AC_LOWER_CASE) || (word_type == AC_INITIAL_CAP)) {
    first_alphanum_index_ = first_alphanum_index (word, lengths);
    first_alphanum_offset_ = first_alphanum_offset (word, lengths);
    if (STRING (conflict_set_I_l_1).contains (word[first_alphanum_offset_])) {
      if (update_map)
        word_res->reject_map[first_alphanum_index_].
            setrej_1Il_conflict ();
      return TRUE;
    }
    else
      return FALSE;
  }
  else if (word_type == AC_UPPER_CASE) {
    return FALSE;
  }
  else {
    if (update_map)
      reject_I_1_L(word_res);
    return TRUE;
  }
}


inT16 first_alphanum_index(const char *word,
                           const char *word_lengths) {
  inT16 i;
  inT16 offset;

  for (i = 0, offset = 0; word[offset] != '\0'; offset += word_lengths[i++]) {
    if (unicharset.get_isalpha(word + offset, word_lengths[i]) ||
        unicharset.get_isdigit(word + offset, word_lengths[i]))
      return i;
  }
  return -1;
}

inT16 first_alphanum_offset(const char *word,
                            const char *word_lengths) {
  inT16 i;
  inT16 offset;

  for (i = 0, offset = 0; word[offset] != '\0'; offset += word_lengths[i++]) {
    if (unicharset.get_isalpha(word + offset, word_lengths[i]) ||
        unicharset.get_isdigit(word + offset, word_lengths[i]))
      return offset;
  }
  return -1;
}

inT16 alpha_count(const char *word,
                  const char *word_lengths) {
  inT16 i;
  inT16 offset;
  inT16 count = 0;

  for (i = 0, offset = 0; word[offset] != '\0'; offset += word_lengths[i++]) {
    if (unicharset.get_isalpha (word + offset, word_lengths[i]))
      count++;
  }
  return count;
}


BOOL8 word_contains_non_1_digit(const char *word,
                                const char *word_lengths) {
  inT16 i;
  inT16 offset;

  for (i = 0, offset = 0; word[offset] != '\0'; offset += word_lengths[i++]) {
    if (unicharset.get_isdigit (word + offset, word_lengths[i]) &&
        (word_lengths[i] != 1 || word[offset] != '1'))
      return TRUE;
  }
  return FALSE;
}


BOOL8 test_ambig_word(  //test for ambiguity
                      WERD_RES *word) {
  BOOL8 ambig = FALSE;

  if ((word->best_choice->permuter () == SYSTEM_DAWG_PERM) ||
    (word->best_choice->permuter () == FREQ_DAWG_PERM) ||
  (word->best_choice->permuter () == USER_DAWG_PERM)) {
    ambig = !NoDangerousAmbig(word->best_choice->string().string(),
                              word->best_choice->lengths().string(),
                              NULL);
  }
  return ambig;
}


/*************************************************************************
 * ambig_word()
 *
 * This is a recursive routine which tests the dictionary for all combinations
 * of conflict set alternatives for characters in a given word.
 *************************************************************************/

BOOL8 ambig_word(                     //original word
                 const char *start_word,
                 char *temp_word,     //alterable copy
                 inT16 test_char_pos  //idx to char to alter
                ) {
  const char *ambigs;            //Ambiguities for char

  if (*(temp_word + test_char_pos) == '\0') {
    if (safe_dict_word (temp_word)) {
      if (strcmp (start_word, temp_word) == 0)
        return FALSE;
      else
        return TRUE;
    }
    else
      return FALSE;
  }
  else {
    ambigs = char_ambiguities (*(temp_word + test_char_pos));
    if (ambigs == NULL)
      return ambig_word (start_word, temp_word, test_char_pos + 1);
    else {
      while (*ambigs != '\0') {
        *(temp_word + test_char_pos) = *ambigs++;
        //test next ambiguity
        if (ambig_word (start_word, temp_word, test_char_pos + 1))
          return TRUE;
      }
      return FALSE;
    }
  }
}


/*************************************************************************
 * char_ambiguities()
 *
 * Return a pointer to a string containing the full conflict set of characters
 * which includes the specified character, if there is one. If the specified
 * character is not a member of a conflict set, return NULL.
 * (NOTE that a character is assumed to be a member of only ONE conflict set.)
 *************************************************************************/

const char *char_ambiguities(char c) {
  static STRING_CLIST conflict_sets;
  static BOOL8 read_conflict_sets = FALSE;
  STRING_C_IT cs_it(&conflict_sets);
  const char *cs;
  STRING cs_file_name;
  FILE *cs_file;
  char buff[1024];

  if (!read_conflict_sets) {
    cs_file_name = datadir + "confsets";
    if (!(cs_file = fopen (cs_file_name.string (), "r"))) {
      CANTOPENFILE.error ("char_ambiguities", EXIT, "%s %d",
        cs_file_name.string (), errno);
    }
    while (fscanf (cs_file, "%s", buff) == 1) {
      cs_it.add_after_then_move (new STRING (buff));
    }
    read_conflict_sets = TRUE;
    cs_it.move_to_first ();
    if (tessedit_rejection_debug) {
      for (cs_it.mark_cycle_pt ();
      !cs_it.cycled_list (); cs_it.forward ()) {
        tprintf ("\"%s\"\n", cs_it.data ()->string ());
      }
    }
  }

  cs_it.move_to_first ();
  for (cs_it.mark_cycle_pt (); !cs_it.cycled_list (); cs_it.forward ()) {
    cs = cs_it.data ()->string ();
    if (strchr (cs, c) != NULL)
      return cs;
  }
  return NULL;
}

#ifndef EMBEDDED
void test_ambigs(const char *word) {
  char orig_word[80];
  char temp_word[80];

  if (strlen (word) > 80)
    tprintf ("Ridiculously long word \"%s\"\n", word);
  else {
    strcpy(orig_word, word);
    while (strlen (orig_word) > 0) {
      strcpy(temp_word, orig_word);

      #ifndef SECURE_NAMES
      if (ambig_word (orig_word, temp_word, 0))
        tprintf ("Ambiguity \"%s\" -> \"%s\"\n", orig_word, temp_word);
      else
        tprintf ("NO Ambiguities for  \"%s\"\n", orig_word);
      tprintf ("Next Word > ");
      #endif
      scanf ("%s", orig_word);
    }
  }
}
#endif

/*************************************************************************
 * dont_allow_dubious_chars()
 * Let Rejects "eat" into adjacent "dubious" chars. I.e those prone to be wrong
 * if adjacent to a reject.
 *************************************************************************/
void dont_allow_dubious_chars(WERD_RES *word) {
  int i = 0;
  int offset = 0;
  int rej_pos;
  int word_len = word->reject_map.length ();

  while (i < word_len) {
    /* Find next reject */

    while ((i < word_len) && (word->reject_map[i].accepted ()))
    {
      offset += word->best_choice->lengths()[i];
      i++;
    }

    if (i < word_len) {
      rej_pos = i;

      /* Reject dubious chars to the left */
      i--;
      offset -= word->best_choice->lengths()[i];
      while ((i >= 0) &&
        STRING (dubious_chars_left_of_reject).contains (word->
        best_choice->
        string ()
      [offset])) {
        word->reject_map[i--].setrej_dubious ();
        offset -= word->best_choice->lengths()[i];
      }

      /* Skip adjacent rejects */

      for (i = rej_pos;
        (i < word_len) && (word->reject_map[i].rejected ());
           offset += word->best_choice->lengths()[i++]);

      /* Reject dubious chars to the right */

      while ((i < word_len) &&
        STRING (dubious_chars_right_of_reject).contains (word->
        best_choice->
        string ()
      [offset])) {
        offset += word->best_choice->lengths()[i];
        word->reject_map[i++].setrej_dubious ();
      }
    }
  }
}


/*************************************************************************
 * dont_allow_1Il()
 * Dont unreject LONE accepted 1Il conflict set chars
 *************************************************************************/
void dont_allow_1Il(WERD_RES *word) {
  int i = 0;
  int offset;
  int word_len = word->reject_map.length ();
  const char *s = word->best_choice->string ().string ();
  const char *lengths = word->best_choice->lengths ().string ();
  BOOL8 accepted_1Il = FALSE;

  for (i = 0, offset = 0; i < word_len;
       offset += word->best_choice->lengths()[i++]) {
    if (word->reject_map[i].accepted ()) {
      if (STRING (conflict_set_I_l_1).contains (s[offset]))
        accepted_1Il = TRUE;
      else {
        if (unicharset.get_isalpha (s + offset, lengths[i]) ||
            unicharset.get_isdigit (s + offset, lengths[i]))
          return;                // >=1 non 1Il ch accepted
      }
    }
  }
  if (!accepted_1Il)
    return;                      //Nothing to worry about

  for (i = 0, offset = 0; i < word_len;
       offset += word->best_choice->lengths()[i++]) {
    if (STRING (conflict_set_I_l_1).contains (s[offset]) &&
      word->reject_map[i].accepted ())
      word->reject_map[i].setrej_postNN_1Il ();
  }
}


inT16 count_alphanums(  //how many alphanums
                      WERD_RES *word) {
  int count = 0;
  int i;
  int offset;

  for (i = 0, offset = 0; i < word->reject_map.length ();
       offset += word->best_choice->lengths()[i++]) {
    if ((word->reject_map[i].accepted ()) &&
      (unicharset.get_isalpha (word->best_choice->string ().string() + offset,
                               word->best_choice->lengths ()[i]) ||
       unicharset.get_isdigit (word->best_choice->string ().string() + offset,
                               word->best_choice->lengths ()[i])))
      count++;
  }
  return count;
}


void reject_mostly_rejects(  //rej all if most rejectd
                           WERD_RES *word) {
  /* Reject the whole of the word if the fraction of rejects exceeds a limit */

  if ((float) word->reject_map.reject_count () / word->reject_map.length () >=
    rej_whole_of_mostly_reject_word_fract)
    word->reject_map.rej_word_mostly_rej ();
}


BOOL8 repeated_nonalphanum_wd(WERD_RES *word, ROW *row) {
  inT16 char_quality;
  inT16 accepted_char_quality;

  if (word->best_choice->lengths ().length () <= 1)
    return FALSE;

  if (!STRING (ok_repeated_ch_non_alphanum_wds).
    contains (word->best_choice->string ()[0]))
    return FALSE;

  if (!repeated_ch_string (word->best_choice->string ().string (),
                           word->best_choice->lengths ().string ()))
    return FALSE;

  word_char_quality(word, row, &char_quality, &accepted_char_quality);

  if ((word->best_choice->lengths ().length () == char_quality) &&
    (char_quality == accepted_char_quality))
    return TRUE;
  else
    return FALSE;
}


BOOL8 repeated_ch_string(const char *rep_ch_str,
                         const char *lengths) {
  UNICHAR_ID c;

  if ((rep_ch_str == NULL) || (*rep_ch_str == '\0')) {
    return FALSE;
  }

  c = unicharset.unichar_to_id(rep_ch_str, *lengths);
  rep_ch_str += *(lengths++);
  while (*rep_ch_str != '\0' &&
         unicharset.unichar_to_id(rep_ch_str, *lengths) == c) {
    rep_ch_str++;
  }
  if (*rep_ch_str == '\0')
    return TRUE;
  return FALSE;
}


inT16 safe_dict_word(const char *s) {
  int dict_word_type;

  dict_word_type = dict_word (s);
  if (dict_word_type == DOC_DAWG_PERM)
    return 0;
  else
    return dict_word_type;
}


void flip_hyphens(WERD_RES *word) {
  char *str = (char *) word->best_choice->string ().string ();
  int i = 0;
  int offset = 0;
  PBLOB_IT outword_it;
  int prev_right = -9999;
  int next_left;
  TBOX out_box;
  float aspect_ratio;

  if (tessedit_lower_flip_hyphen <= 1)
    return;

  outword_it.set_to_list (word->outword->blob_list ());

  for (outword_it.mark_cycle_pt ();
  !outword_it.cycled_list (); outword_it.forward (),
           offset += word->best_choice->lengths()[i++]) {
    out_box = outword_it.data ()->bounding_box ();
    if (outword_it.at_last ())
      next_left = 9999;
    else
      next_left = outword_it.data_relative (1)->bounding_box ().left ();
    /*
      Dont touch small or touching blobs - it is too dangerous
    */
    if ((out_box.width () > 8 * word->denorm.scale ()) &&
    (out_box.left () > prev_right) && (out_box.right () < next_left)) {
      aspect_ratio = out_box.width () / (float) out_box.height ();
      if (str[offset] == '.') {
        if (aspect_ratio >= tessedit_upper_flip_hyphen) {
          /* Certain HYPHEN */
          str[offset] = '-';
          if (word->reject_map[i].rejected ())
            word->reject_map[i].setrej_hyphen_accept ();
        }
        if ((aspect_ratio > tessedit_lower_flip_hyphen) &&
          word->reject_map[i].accepted ())
                                 //Suspected HYPHEN
          word->reject_map[i].setrej_hyphen ();
      }
      else if (str[offset] == '-') {
        if ((aspect_ratio >= tessedit_upper_flip_hyphen) &&
          (word->reject_map[i].rejected ()))
          word->reject_map[i].setrej_hyphen_accept ();
        //Certain HYPHEN

        if ((aspect_ratio <= tessedit_lower_flip_hyphen) &&
          (word->reject_map[i].accepted ()))
                                 //Suspected HYPHEN
          word->reject_map[i].setrej_hyphen ();
      }
    }
    prev_right = out_box.right ();
  }
}


void flip_0O(WERD_RES *word) {
  char *str = (char *) word->best_choice->string ().string ();
  char *lengths = (char *) word->best_choice->lengths ().string ();
  int i;
  int offset;
  PBLOB_IT outword_it;
  TBOX out_box;

  if (!tessedit_flip_0O)
    return;

  outword_it.set_to_list (word->outword->blob_list ());

  for (i = 0, offset = 0, outword_it.mark_cycle_pt ();
  !outword_it.cycled_list (); offset += lengths[i++], outword_it.forward ()) {
    if (unicharset.get_isupper (str + offset, lengths[i]) ||
        unicharset.get_isdigit (str + offset, lengths[i])) {
      out_box = outword_it.data ()->bounding_box ();
      if ((out_box.top () < bln_baseline_offset + bln_x_height) ||
        (out_box.bottom () > bln_baseline_offset + bln_x_height / 4))
        return;                  //Beware words with sub/superscripts
    }
  }

  for (i = 1, offset = lengths[0]; str[offset] != '\0';
       offset += lengths[i++], outword_it.forward ()) {
    if (lengths[i] == 1 &&
        ((str[offset] == '0') || (str[offset] == 'O'))) {
      /* A0A */
      if (non_O_upper (str + offset - lengths[i - 1], lengths[i - 1]) &&
          lengths[i + 1] > 0 &&
          non_O_upper (str + offset + lengths[i], lengths[i + 1])) {
        str[offset] = 'O';
      }
      /* A00A */
      if (non_O_upper (str + offset - lengths[i - 1], lengths[i - 1]) &&
        ((lengths[i + 1] == 1 && str[offset + lengths[i]] == '0') ||
         (lengths[i + 1] == 1 && str[offset + lengths[i]] == 'O')) &&
          lengths[i + 2] > 0 &&
          non_O_upper (str + offset + lengths[i] + lengths[i + 1],
                       lengths[i + 2])) {
        str[offset] = 'O';
        str[offset + lengths[i]] = 'O';
        offset += lengths[i++];
      }
      /* AA0<non digit or end of word> */
      if ((i > 1) &&
        non_O_upper (str + offset - lengths[i - 1] - lengths[i - 2],
                     lengths[i - 2]) &&
        non_O_upper (str + offset - lengths[i - 1], lengths[i - 1]) &&
          lengths[i + 1] > 0 &&
        !unicharset.get_isdigit (str + offset + lengths[i], lengths[i + 1]) &&
          (lengths[i + 1] != 1 || str[offset + lengths[i]] != 'l') &&
          (lengths[i + 1] != 1 || str[offset + lengths[i]] != 'I')) {
        str[offset] = 'O';
      }
      /* 9O9 */
      if (non_0_digit (str + offset - lengths[i - 1], lengths[i - 1]) &&
          lengths[i + 1] > 0 &&
          non_0_digit (str + offset + lengths[i], lengths[i + 1])) {
        str[offset] = '0';
      }
      /* 9OOO */
      if (non_0_digit (str + offset - lengths[i - 1], lengths[i - 1]) &&
          ((lengths[i + 1] == 1 && str[offset + lengths[i]] == '0') ||
           (lengths[i + 1] == 1 && str[offset + lengths[i]] == 'O')) &&
          ((lengths[i + 2] == 1 &&
            str[offset + lengths[i] + lengths[i + 1]] == '0') ||
           (lengths[i + 2] == 1 &&
            str[offset + lengths[i] + lengths[i + 1]] == 'O'))) {
        str[offset] = '0';
        str[offset + lengths[i]] = '0';
        str[offset + lengths[i] + lengths[i + 1]] = '0';
        offset += lengths[i++];
        offset += lengths[i++];
      }
      /* 9OO<non upper> */
      if (non_0_digit (str + offset - lengths[i - 1], lengths[i - 1]) &&
          ((lengths[i + 1] == 1 && str[offset + lengths[i]] == '0') ||
           (lengths[i + 1] == 1 && str[offset + lengths[i]] == 'O')) &&
          lengths[i + 2] > 0 &&
          !unicharset.get_isupper (str + offset + lengths[i] + lengths[i + 1],
                                   lengths[i + 2])) {
        str[offset] = '0';
        str[offset + lengths[i]] = '0';
        offset += lengths[i++];
      }
      /* 9O<non upper> */
      if (non_0_digit (str + offset - lengths[i - 1], lengths[i - 1]) &&
          lengths[i + 1] > 0 &&
          !unicharset.get_isupper (str + offset + lengths[i], lengths[i + 1])) {
        str[offset] = '0';
      }
      /* 9[.,]OOO.. */
      if ((i > 1) &&
        ((lengths[i - 1] == 1 && str[offset - lengths[i - 1]] == '.') ||
         (lengths[i - 1] == 1 && str[offset - lengths[i - 1]] == ',')) &&
          (unicharset.get_isdigit (str + offset -
                                   lengths[i - 1] - lengths[i - 2],
                                   lengths[i - 2]) ||
           (lengths[i - 2] == 1 &&
            str[offset - lengths[i - 1] - lengths[i - 2]] == 'O'))) {
        if (lengths[i - 2] == 1 &&
            str[offset - lengths[i - 1] - lengths[i - 2]] == 'O')
          str[offset - lengths[i - 1] - lengths[i - 2]] = '0';
        while (lengths[i] == 1 &&
               ((str[offset] == 'O') || (str[offset] == '0'))) {
          str[offset] = '0';
          offset += lengths[i++];
        }
        i--;
        offset -= lengths[i];
      }
    }
  }
}


BOOL8 non_O_upper(const char* str, int length) {
  return unicharset.get_isupper (str, length) &&
      (!unicharset.eq(unicharset.unichar_to_id(str, length), "O"));
}


BOOL8 non_0_digit(const char* str, int length) {
  return unicharset.get_isdigit (str, length) &&
      (!unicharset.eq(unicharset.unichar_to_id(str, length), "0"));
}
