/*
 *****************************************************************************
 *
 *	$rcsfile: lexutil.l,v $
 *	$Date: 1999/08/16 18:20:58 $
 *	$Source: /home/richard/Xml/RCS/lexutil.l,v $
 *	$Revision: 1.141 $
 *	$Author: richard $
 *
 *****************************************************************************
 *
 *  Copyright 1998, 1999 Brown University and Richard Goerwitz
 *
 *****************************************************************************
 *
 *  Lexer for xmlparse.  Goes with parsutil.y.  Note that this file
 *  will not work with ordinary Lex.  Nor will it work with versions
 *  of flex that cannot handle Unicode. See the INSTALL file for
 *  more details, and for information on obtaining Unicode patches for
 *  flex.
 *
 *  Note well: To view this file productively, you'll need to set your
 *  editor to show at least 100 columns of text.
 *
 *****************************************************************************
 *
 *  This file contains specifications for recognizing basic XML language
 *  tokens, including:
 *
 *   - XML declarations and processing instructions
 *   - character and general/parameter entity references
 *   - doctype and other DTD-related markup declarations
 *   - general markup, quoted strings, comments
 *
 *  Like most (f)lex specifications, this one contains three sections.
 *  They hold the following items:
 *
 *   1) The definitions section contains specifications for Unicode
 *      character classes that form part of the XML spec
 *   2) The rule section contains patterns for recognizing markup,
 *      processing instructions, etc. (as noted above)
 *   3) The user code section contains state stack manipulation and
 *      stream-switching utilities
 *
 *  When reading the code below, several points should be kept in
 *  mind:
 *
 *   - Global variables all have an xml_yy prefix (e.g., xml_yytext)
 *   - Extensive use is made of state stacks and state-stack utility
 *     routines such as yy_push/pop_state()
 *   - Extensive use is made of Flex's stream-switching capabilities
 *
 *  This last point requires some elaboration.
 *
 *  XML makes extensive use of PUBLIC/SYSTEM identifiers, which point
 *  to external files that must generally be read in and parsed.  In
 *  order to parse these files, the parser (in parsutil.y) pushes a new
 *  input source onto a stack (xml_file_stack) using push_xml_file(),
 *  push_entity_text(), or push_xml_string().  Often these routines
 *  will also push lexer states using yy_push_state().
 *
 *  Note also: This lexer uses a simple string-allocation and garbage
 *  collection mechanism.  To strdup and to return a pointer to the
 *  newly allocated space, use uni_add_string().  To free all strings
 *  so allocated, use uni_free_strings() (this gets done after yyparse
 *  returns).
 *
 *****************************************************************************
 */

%{
#include "general.h"
#include "grammutil.h"
#include "parsutil.h"
#include "utfutil.h"

struct xml_file_state
{
  YY_BUFFER_STATE buffer;
  struct xml_file *xf;
  enum scan_objects what_object;
  int lexer_stack_marker;
};
/* used for pushing dummy states onto the state stack */
#define MARKER_BASE	40000

static int eof_count = 0;
static int bad_char_count = 0;
static int xml_file_stack_len = 0;
static int xml_file_stack_buflen = 0;
static struct xml_file_state *xml_file_stack = NULL;

static int pushback (my_wchar_t *);
static int push_xml_string (struct xml_file *, my_wchar_t *);
static int check_parend_nesting (my_wchar_t *, my_wchar_t *);

#define STATECOUNT 10
/* used to hold line-number starting points for markup, xmldecl, etc. */
enum states { markup, xmldecl, pistuff, commstuff, decl, dtd, include, ignore, marked };
static int starting_points[STATECOUNT] = { 0 };

%}

/* control characters minus nil, HT, CR, and LF (not used, for the most part) */
Control		[\x01-\x08\x0b-\x0c\x0e-\x1f]

/* Unicode digits */
Digit           [\x0030-\x0039\x0660-\x0669\x06f0-\x06f9\x0966-\x096f\x09e6-\x09ef\x0a66-\x0a6f\x0ae6-\x0aef\x0b66-\x0b6f\x0be6-\x0bef\x0c66-\x0c6f\x0ce6-\x0cef\x0d66-\x0d6f\x0e50-\x0e59\x0ed0-\x0ed9\x0f20-\x0f29]

/* The XML standard defines this; doesn't use it right */
Char		[\x09\x0a\x0d\x20-\xfffd]

/* Used in the comment production - all chars except '-' */
CommentChar	[\x09\x0a\x0d\x20-\x2c\x2e-\xfffd]
CommentCharNoGt	[\x09\x0a\x0d\x20-\x2c\x2e-\x3d\x3f-\xfffd]
CommentCharNoGtOrLt [\x09\x0a\x0d\x20-\x2c\x2e-\x3b\x3d\x3f-\xfffd]

/* Everything but the left square bracket, ] (\x5d) */
CDChar		[\x09\x0a\x0d\x20-\x5c\x5e-\xfffd]
CDCharNoGt	[\x09\x0a\x0d\x20-\x3d\x3f-\x5c\x5e-\xfffd]

/* Everything but the greater-than sign */
CharNoGt	[\x09\x0a\x0d\x20-\x3d\x3f-\xfffd]

/* Everything but the less-than sign, &, and whitespace */
CharNoLtOrAmpOrSpace [\x21-\x25\x27-\x3b\x3d-\xfffd]
CharNoLtOrGtOrAmpOrPercentOrLpOrQuoteOrSpace [\x21\x24\x29-\x3a\x40-\xfffd]

/* Everything but the question mark, ? (\x3f) */
PiChar		[\x09\x0a\x0d\x20-\x3e\x40-\xfffd]
PiCharNoGt	[\x09\x0a\x0d\x20-\x3d\x40-\xfffd]
PiCharNoGtOrLt	[\x09\x0a\x0d\x20-\x3b\x3d\x40-\xfffd]

/* Unicode ideographic characters */
Ideographic	[\x3007\x3021-\x3029\x4e00-\x9fa5]

/* I have no idea what these are for */
Extender	[\x00b7\x02d0\x02d1\x0387\x0640\x0e46\x0ec6\x3005\x3031-\x3035\x309d-\x309e\x30fc-\x30fe]

/* Unicode combining characters (diacritics; Hebrew vowel letters,
 * etc.; these go with base characters)
 */
CombiningChar	[\x0300-\x0345\x0360-\x0361\x0483-\x0486\x0591-\x05a1\x05a3-\x05b9\x05bb-\x05bd\x05bf\x05c1-\x05c2\x05c4\x064b-\x0652\x0670\x06d6-\x06dc\x06dd-\x06df\x06e0-\x06e4\x06e7-\x06e8\x06ea-\x06ed\x0901-\x0903\x093c\x093e-\x094c\x094d\x0951-\x0954\x0962-\x0963\x0981-\x0983\x09bc\x09be\x09bf\x09c0-\x09c4\x09c7-\x09c8\x09cb-\x09cd\x09d7\x09e2-\x09e3\x0a02\x0a3c\x0a3e\x0a3f\x0a40-\x0a42\x0a47-\x0a48\x0a4b-\x0a4d\x0a70-\x0a71\x0a81-\x0a83\x0abc\x0abe-\x0ac5\x0ac7-\x0ac9\x0acb-\x0acd\x0b01-\x0b03\x0b3c\x0b3e-\x0b43\x0b47-\x0b48\x0b4b-\x0b4d\x0b56-\x0b57\x0b82-\x0b83\x0bbe-\x0bc2\x0bc6-\x0bc8\x0bca-\x0bcd\x0bd7\x0c01-\x0c03\x0c3e-\x0c44\x0c46-\x0c48\x0c4a-\x0c4d\x0c55-\x0c56\x0c82-\x0c83\x0cbe-\x0cc4\x0cc6-\x0cc8\x0cca-\x0ccd\x0cd5-\x0cd6\x0d02-\x0d03\x0d3e-\x0d43\x0d46-\x0d48\x0d4a-\x0d4d\x0d57\x0e31\x0e34-\x0e3a\x0e47-\x0e4e\x0eb1\x0eb4-\x0eb9\x0ebb-\x0ebc\x0ec8-\x0ecd\x0f18-\x0f19\x0f35\x0f37\x0f39\x0f3e\x0f3f\x0f71-\x0f84\x0f86-\x0f8b\x0f90-\x0f95\x0f97\x0f99-\x0fad\x0fb1-\x0fb7\x0fb9\x20d0-\x20dc\x20e1\x302a-\x302f\x3099\x309a]

/* Unicode base characters (A, B, C, etc.; may be followed by things
 * like combining characters).  Starts at ASCII capital A (\x0041).
 */
BaseChar	[\x0041-\x005a\x0061-\x007a\x00c0-\x00d6\x00d8-\x00f6\x00f8-\x00ff\x0100-\x0131\x0134-\x013e\x0141-\x0148\x014a-\x017e\x0180-\x01c3\x01cd-\x01f0\x01f4-\x01f5\x01fa-\x0217\x0250-\x02a8\x02bb-\x02c1\x0386\x0388-\x038a\x038c\x038e-\x03a1\x03a3-\x03ce\x03d0-\x03d6\x03da\x03dc\x03de\x03e0\x03e2-\x03f3\x0401-\x040c\x040e-\x044f\x0451-\x045c\x045e-\x0481\x0490-\x04c4\x04c7-\x04c8\x04cb-\x04cc\x04d0-\x04eb\x04ee-\x04f5\x04f8-\x04f9\x0531-\x0556\x0559\x0561-\x0586\x05d0-\x05ea\x05f0-\x05f2\x0621-\x063a\x0641-\x064a\x0671-\x06b7\x06ba-\x06be\x06c0-\x06ce\x06d0-\x06d3\x06d5\x06e5-\x06e6\x0905-\x0939\x093d\x0958-\x0961\x0985-\x098c\x098f-\x0990\x0993-\x09a8\x09aa-\x09b0\x09b2\x09b6-\x09b9\x09dc-\x09dd\x09df-\x09e1\x09f0-\x09f1\x0a05-\x0a0a\x0a0f-\x0a10\x0a13-\x0a28\x0a2a-\x0a30\x0a32-\x0a33\x0a35-\x0a36\x0a38-\x0a39\x0a59-\x0a5c\x0a5e\x0a72-\x0a74\x0a85-\x0a8b\x0a8d\x0a8f-\x0a91\x0a93-\x0aa8\x0aaa-\x0ab0\x0ab2-\x0ab3\x0ab5-\x0ab9\x0abd\x0ae0\x0b05-\x0b0c\x0b0f-\x0b10\x0b13-\x0b28\x0b2a-\x0b30\x0b32-\x0b33\x0b36-\x0b39\x0b3d\x0b5c-\x0b5d\x0b5f-\x0b61\x0b85-\x0b8a\x0b8e-\x0b90\x0b92-\x0b95\x0b99-\x0b9a\x0b9c\x0b9e-\x0b9f\x0ba3-\x0ba4\x0ba8-\x0baa\x0bae-\x0bb5\x0bb7-\x0bb9\x0c05-\x0c0c\x0c0e-\x0c10\x0c12-\x0c28\x0c2a-\x0c33\x0c35-\x0c39\x0c60-\x0c61\x0c85-\x0c8c\x0c8e-\x0c90\x0c92-\x0ca8\x0caa-\x0cb3\x0cb5-\x0cb9\x0cde\x0ce0-\x0ce1\x0d05-\x0d0c\x0d0e-\x0d10\x0d12-\x0d28\x0d2a-\x0d39\x0d60-\x0d61\x0e01-\x0e2e\x0e30\x0e32-\x0e33\x0e40-\x0e45\x0e81-\x0e82\x0e84\x0e87-\x0e88\x0e8a\x0e8d\x0e94-\x0e97\x0e99-\x0e9f\x0ea1-\x0ea3\x0ea5\x0ea7\x0eaa-\x0eab\x0ead-\x0eae\x0eb0\x0eb2-\x0eb3\x0ebd\x0ec0-\x0ec4\x0f40-\x0f47\x0f49-\x0f69\x10a0-\x10c5\x10d0-\x10f6\x1100\x1102-\x1103\x1105-\x1107\x1109\x110b-\x110c\x110e-\x1112\x113c\x113e\x1140\x114c\x114e\x1150\x1154-\x1155\x1159\x115f-\x1161\x1163\x1165\x1167\x1169\x116d-\x116e\x1172-\x1173\x1175\x119e\x11a8\x11ab\x11ae-\x11af\x11b7-\x11b8\x11ba\x11bc-\x11c2\x11eb\x11f0\x11f9\x1e00-\x1e9b\x1ea0-\x1ef9\x1f00-\x1f15\x1f18-\x1f1d\x1f20-\x1f45\x1f48-\x1f4d\x1f50-\x1f57\x1f59\x1f5b\x1f5d\x1f5f-\x1f7d\x1f80-\x1fb4\x1fb6-\x1fbc\x1fbe\x1fc2-\x1fc4\x1fc6-\x1fcc\x1fd0-\x1fd3\x1fd6-\x1fdb\x1fe0-\x1fec\x1ff2-\x1ff4\x1ff6-\x1ffc\x2126\x212a-\x212b\x212e\x2180-\x2182\x3041-\x3094\x30a1-\x30fa\x3105-\x312c\xac00-\xd7a3]

Letter		{BaseChar}|{Ideographic}
NameChar	{Letter}|{Digit}|\.|-|_|:|{CombiningChar}|{Extender}

/* xml_yytext instead of yytext, etc. */
%option prefix="xml_yy"

/* we'll be using start-condition stacks */
%option stack

/* make sure we don't look ahead farther than necessary */
%option always-interactive

/* exclusive states; ain't flex cool? */
%x MARKUP XMLDECL PISTUFF COMMSTUFF DTD DECL IGNORE IGNORE_DECL IGNORE_MARKED_SECT

%%

<*>\xFFFE|\xFFFF			{ /* FFFE and FFFF are NOT Unicode characters.  FFFE
					   * is used to detect byte order; if we see it first
					   * thing in a file, it tells the processor to swap
					   * bytes.  We should never see it here.
					   */
					  add_xml_error (this_file, 1057, xml_yytext);
					}

]*]]>					{ add_xml_error (this_file, 1300, xml_yytext); }

{CharNoLtOrAmpOrSpace}+			{ my_wchar_t *wp;
					  if ((wp = uni_strchr (xml_yytext, ']')))
					    if (uni_utf_strstr (wp, "]]>"))
					      /* see the above pattern */
					      add_xml_error (this_file, 1300, xml_yytext);
					  xml_yylval.wstr = uni_add_string (xml_yytext);
					  return CHAR_DATA;
					}
<INITIAL,DTD>[\x09\x0d\x0a\x20]+<\?[\x09\x0d\x0a\x20]*([Xx][Mm][Ll]) {
					  my_wchar_t c, *wp;
					  /* only perform check on main document and
					   * its external DTD entity (if there is one);
					   * read_entire_xml_file() will handle other
					   * files */
					  for (wp = xml_yytext; *wp && *wp != '<'; wp++);
					  if (validating (this_file) && this_file->lineno <= 1) {
					    c = *wp; *wp = 0;
					    add_xml_error (this_file, 386, xml_yytext);
					    *wp = c;
					  }
					  pushback (wp);
					}
[\x09\x0d\x0a\x20]+			{ xml_yylval.wstr = uni_add_string (xml_yytext);
					  return WHITESPACE;
					}
&&					{ xml_yylval.wstr = uni_add_string (xml_yytext);
					  return CHAR_DATA;
					}
&<					{ xml_yylval.wstr = uni_add_string (xml_yytext);
					  return CHAR_DATA;
					}
&#x?0+;					{ add_xml_error (this_file, 1051, xml_yytext); }
&({Digit})+;				{ add_xml_error (this_file, 1052, xml_yytext); }
&({Digit})+				{ add_xml_error (this_file, 1053, xml_yytext); }
&(#({Digit})+|#x[0-9a-fA-F]+);		{ my_wchar_t *expansion;
					  if (! validating (this_file)) {
					    xml_yylval.wstr = uni_add_string (xml_yytext);
					    return CHAR_DATA;
					  }
					  else {
					    if (! in_document_content && what_are_we_scanning () != file)
					      add_xml_error (this_file, 1050, xml_yytext);
					    /* attempt to expand entity */
					    expansion = map_entities (this_file, xml_yytext,
					      MAP_CHAR_ENTITIES | MAP_AMP_AND_LT, 0);
					    if (expansion == NULL) {
					      add_xml_error (this_file, 1003, xml_yytext);
					      xml_yylval.wstr = uni_add_string (xml_yytext);
					      return BAD_ENTITY_REF;
					    } else {
					      xml_yylval.wstr = uni_add_string (expansion);
					      /* map_entities returns a malloc'd string */
					      free (expansion);
					      return CHAR_DATA;
					    }
					  }
					}
&(({Letter})|_|:)({NameChar})*;		{ my_wchar_t *expansion;
					  if (! validating (this_file)) {
					    xml_yylval.wstr = uni_add_string (xml_yytext);
					    return CHAR_DATA;
					  }
					  else {
					    if (! in_document_content && what_are_we_scanning () == file)
					      /* entity refs must be in document content */
					      add_xml_error (this_file, 1050, xml_yytext);
					    /* recursively expand entity */
					    expansion = map_entities (this_file, xml_yytext,
					      MAP_GENERAL_ENTITIES | ABORT_ON_FAILURE, 0);
					    if (expansion == NULL) {
					      add_xml_error (this_file, 1003, xml_yytext);
					      xml_yylval.wstr = uni_add_string (xml_yytext);
					      return BAD_ENTITY_REF;
					    } else {
					      /* rescan expansion */
					      push_xml_string (this_file, expansion);
					      /* map_entities returns a malloc'd string */
					      free (expansion);
					    }
					  }
					}
&[^;<\x09\x0d\x0a\x20]*			{ add_xml_error (this_file, 1004, xml_yytext);
					  xml_yylval.wstr = uni_add_string (xml_yytext);
					  return BAD_ENTITY_REF;
					}

\<!					{ my_wchar_t *wp;
					  /* ignore spurious '!'; report possible markup in doc body */
					  add_xml_error (this_file, 1080, xml_yytext);
					  wp = uni_strdup (utf_8_to_utf_16 ("<"));
					  pushback (wp);
					  free (wp);
					}
\<\%					{ my_wchar_t *wp;
					  /* ignore spurious '%' */
					  add_xml_error (this_file, 1081, xml_yytext);
					  wp = uni_strdup (utf_8_to_utf_16 ("<"));
					  pushback (wp);
					  free (wp);
					}
\<DOCTYPE				{ if (this_file->child == no) {
					    /* Try to catch bogus DOCTYPE decls w/ no ! */
					    if (this_file->lineno <= 2)
					      add_xml_warning (this_file, 405, xml_yytext);
					  }
					  yy_push_state (MARKUP);
					  /* set global var to indicate we're in document content */
					  if (! in_document_content && this_file->child == no
					      && what_are_we_scanning () != entity_replacement_text)
					    in_document_content = 1;
					  starting_points[markup] = this_file->lineno;
                                          xml_yylval.wstr = uni_add_string (&xml_yytext[1]);
                                          return ELEMENT;
					}

\<[\x09\x0d\x0a\x20]*(({Letter})|_|:)({NameChar})* {
					  size_t i;
					  yy_push_state (MARKUP);
					  /* set global var to indicate we're in document content */
					  if (! in_document_content && this_file->child == no
					      && what_are_we_scanning () != entity_replacement_text)
					    in_document_content = 1;
					  starting_points[markup] = this_file->lineno;
					  if (uni_isspace (xml_yytext[1]))
					    add_xml_error (this_file, 1115,
					      uni_truncate_to (xml_yytext, 20));
					  for (i = 1; uni_isspace (xml_yytext[i]); i++);
					  xml_yylval.wstr = uni_add_string (&xml_yytext[i]);
					  return ELEMENT;
					}
\<\/[\x09\x0d\x0a\x20]*			{ int i;
					  yy_push_state (MARKUP);
					  starting_points[markup] = this_file->lineno;
					  /* an empty </> tag; too bad they didn't use this in XML */
					  add_xml_error (this_file, 1111, NULL);
					  for (i = 2; uni_isspace (xml_yytext[i]); i++);
					  xml_yylval.wstr = uni_add_string (&xml_yytext[i]);
					  return ETAG;
					}
\<[^\x09\x0d\x0a\x20>?!\/]+({NameChar})+ {
					  size_t i;
					  /* element name (GI) begins with illegal char(s) */
					  add_xml_error (this_file, 1123, xml_yytext);
					  yy_push_state (MARKUP);
					  starting_points[markup] = this_file->lineno;
					  /* set global var to indicate we're in document content */
					  if (! in_document_content && this_file->child == no
					      && what_are_we_scanning () != entity_replacement_text)
					    in_document_content = 1;
					  if (uni_isspace (xml_yytext[1]))
					    add_xml_error (this_file, 1115,
					      uni_truncate_to (xml_yytext, 20));
					  for (i = 1; uni_isspace (xml_yytext[i]); i++);
					  xml_yylval.wstr = uni_add_string (&xml_yytext[i]);
					  return ELEMENT;
					}
\<\/[\x09\x0d\x0a\x20]*(({Letter})|_|:)({NameChar})* {
					  size_t i;
					  yy_push_state (MARKUP);
					  starting_points[markup] = this_file->lineno;
					  if (uni_isspace (xml_yytext[2]))
					    add_xml_error (this_file, 1115,
					      uni_truncate_to (xml_yytext, 20));
					  for (i = 2; uni_isspace (xml_yytext[i]); i++);
					  xml_yylval.wstr = uni_add_string (&xml_yytext[i]);
					  return ETAG;
					}
\<\/[^\x09\x0d\x0a\x20>?!]+({NameChar})+ {
					  size_t i;
					  /* element name (GI) begins with illegal char(s) */
					  add_xml_error (this_file, 1123, xml_yytext);
					  yy_push_state (MARKUP);
					  starting_points[markup] = this_file->lineno;
					  if (uni_isspace (xml_yytext[2]))
					    add_xml_error (this_file, 1115,
					      uni_truncate_to (xml_yytext, 20));
					  for (i = 2; uni_isspace (xml_yytext[i]); i++);
					  xml_yylval.wstr = uni_add_string (&xml_yytext[i]);
					  return ETAG;
					}

<MARKUP,DECL>#[Cc][Oo][Nn][Rr][Ee][Ff]	{ xml_yylval.wstr = uni_add_string (xml_yytext);
					  return CONREF;
					}
<MARKUP,DECL>#[Cc][Uu][Rr][Rr][Ee][Nn][Tt] {
					  xml_yylval.wstr = uni_add_string (xml_yytext);
					  return CURRENT;
					}
<MARKUP>\/[\x09\x0d\x0a\x20]+>		{ yy_pop_state ();
					  xml_yytext[1] = 0;
					  /* spurious whitespace after / in "/ >" */
					  add_xml_error (this_file, 1114, xml_yytext);
					  return EMPTY_ELEMENT_END_DELIM;
					}
<MARKUP>\/>				{ yy_pop_state ();
					  return EMPTY_ELEMENT_END_DELIM;
					}
<MARKUP>>				{ yy_pop_state ();
					  return ELEMENT_OR_ETAG_END_DELIM;
					}


\<[\x09\x0d\x0a\x20]+\/			{ my_wchar_t *wp;
					  xml_yytext[1] = 0;
					  /* spurious whitespace after < in "< /" */
					  add_xml_error (this_file, 1114, xml_yytext);
					  wp = uni_strdup (utf_8_to_utf_16 ("</"));
					  pushback (wp);
					  free (wp);
					}
<INITIAL,DTD><[\x09\x0d\x0a\x20]+\?	{ my_wchar_t *wp;
					  xml_yytext[1] = 0;
					  /* spurious whitespace after < in "< ?" */
					  add_xml_error (this_file, 1114, xml_yytext);
					  wp = uni_strdup (utf_8_to_utf_16 ("<?"));
					  pushback (wp);
					  free (wp);
					}
<INITIAL,DTD><\?(({Letter})|_|:)({NameChar})* {
					  size_t i;
					  xml_yylval.wstr = uni_add_string (&xml_yytext[2]);
					  if (uni_utf_strcasecmp (xml_yylval.wstr, "xml") != 0) {
					    /* not an <?xml decl; just a regular PI */
					    if (uni_strchr (xml_yylval.wstr, ':'))
					      /* colon in PI target discouraged just for now */
					      add_xml_warning (this_file, 390, xml_yylval.wstr);
					    yy_push_state (PISTUFF);
					    starting_points[pistuff] = this_file->lineno;
					    return PI_TARGET;
					  }
					  else {
					    if (this_file->lineno <= 1) {
					      for (i = 0; xml_yytext[i] != 0; i++) {
						if (uni_utf_any (&xml_yytext[i], "XML")) {
						  /* "XML" is supposed to be lower case */
						  add_xml_error (this_file, 365, xml_yylval.wstr);
						  uni_downcase (xml_yytext);
						  break;
						}
					      }
					      /* if we're on line one, <?xml may be okay */
					      yy_push_state (XMLDECL);
					      starting_points[xmldecl] = this_file->lineno;
					      return XMLSTART_DELIM;
					    }
					    if (what_are_we_scanning () != file)
					      /* XML decl should only occur literally (== file) */
					      add_xml_error (this_file, 367, xml_yylval.wstr);
					    /* <?xml is clearly invalid if not on line one */
					    add_xml_error (this_file, 363, uni_truncate_to (xml_yytext, 20));
					    starting_points[pistuff] = this_file->lineno;
					    yy_push_state (PISTUFF);
					    return PI_TARGET;
					  }
					}
\<\?[\x09\x0d\x0a\x20]+			{ my_wchar_t *wp;
					  /* superfluous (and in fact invalid) whitespace */
					  add_xml_error (this_file, 501, uni_truncate_to (xml_yytext, 20));
					  xml_yytext[2] = 0;
					  wp = uni_strdup (xml_yytext);
					  pushback (wp);
					  free (wp);
					}
<XMLDECL>VERSION			{ xml_yylval.wstr = uni_add_string (xml_yytext);
					  add_xml_error (this_file, 365, xml_yytext);
					  return VERSION;
					}
<XMLDECL>version			{ xml_yylval.wstr = uni_add_string (xml_yytext);
					  return VERSION;
					}
<XMLDECL>ENCODING			{ xml_yylval.wstr = uni_add_string (xml_yytext);
					  add_xml_error (this_file, 365, xml_yytext);
					  return ENCODING;
					}
<XMLDECL>encoding			{ xml_yylval.wstr = uni_add_string (xml_yytext);
					  return ENCODING;
					}
<XMLDECL>STANDALONE			{ xml_yylval.wstr = uni_add_string (xml_yytext);
					  add_xml_error (this_file, 365, xml_yytext);
					  return STANDALONE;
					}
<XMLDECL>standalone			{ xml_yylval.wstr = uni_add_string (xml_yytext);
					  return STANDALONE;
					}
<XMLDECL>\?[\x09\x0d\x0a\x20]+>		{ yy_pop_state ();
					  xml_yytext[1] = 0;
					  /* spurious whitespace after ? in "? >" */
					  add_xml_error (this_file, 1114, xml_yytext);
					  return XML_DECL_END_DELIM;
					}
<XMLDECL>\?+>				{ yy_pop_state ();
					  if (uni_utf_strncmp (xml_yytext, "??", 2) == 0)
					    /* XML decl shouldn't end with "??>", etc. */
					    add_xml_error (this_file, 373, xml_yytext);
					  return XML_DECL_END_DELIM;
					}

<INITIAL,DTD><\?[\x09\x0d\x0a\x20]+	{ my_wchar_t *wp;
					  add_xml_error (this_file, 501, xml_yytext);
					  xml_yytext[2] = 0;
					  wp = uni_strdup (xml_yytext);
					  pushback (wp);
					  free (wp);
					}
<INITIAL,DTD><\?[^?>\x09\x0d\x0a\x20]*	{ xml_yylval.wstr = uni_add_string (&xml_yytext[2]);
					  starting_points[pistuff] = this_file->lineno;
					  add_xml_error (this_file, 354, xml_yylval.wstr);
					  yy_push_state (PISTUFF);
					  return PI_TARGET;
					}
<PISTUFF>(({PiCharNoGtOrLt})|\?+({PiCharNoGtOrLt}))* {
					  my_wchar_t *wp, *tmp;
					  yy_pop_state ();
					  if (! starting_points[pistuff]) {
					    tmp = utf_8_to_utf_16 ("<?...");
					    add_xml_error (this_file, 351, tmp);
					  }
					  else {
					    /* line numbers are messed up; reset them */
					    this_file->lineno = starting_points[pistuff];
					    tmp = utf_8_to_utf_16 ("<?...");
					    add_xml_error (this_file, 351, tmp);
					    wp = xml_yytext;
					    while ((wp = uni_strchr (wp, '\n'))) {
					      wp++; this_file->lineno++;
					    }
					  }
					  xml_yytext[xml_yyleng - 1] = 0;
					  xml_yylval.wstr = uni_add_string (xml_yytext);
					  return PIDATA;
					}
<PISTUFF>(({PiChar})|\?+({PiCharNoGt}))*\?+> {
					  yy_pop_state ();
					  xml_yytext[xml_yyleng - 2] = 0;
					  xml_yylval.wstr = uni_add_string (xml_yytext);
					  return PIDATA;
					}


<INITIAL,DTD><![\x09\x0d\x0a\x20]+--	{ my_wchar_t *wp;
					  xml_yytext[2] = 0;
					  /* spurious whitespace after < in "<! --" */
					  add_xml_error (this_file, 1114, xml_yytext);
					  wp = uni_strdup (utf_8_to_utf_16 ("<!--"));
					  pushback (wp);
					  free (wp);
					}
<INITIAL,DTD><!--			{ /* comment can occur in DTD or in text */
					  starting_points[commstuff] = this_file->lineno;
					  yy_push_state (COMMSTUFF);
					  return COMMENT_DELIM;
					}
<COMMSTUFF>({CommentCharNoGtOrLt}|-({CommentCharNoGtOrLt})|--+({CommentCharNoGtOrLt}))+ {
					  my_wchar_t *wp, *tmp;
					  /* unterminated comment */
					  yy_pop_state ();
					  if (! starting_points[commstuff]) {
					    tmp = utf_8_to_utf_16 ("<!--...");
					    add_xml_error (this_file, 903, tmp);
					  }
					  else {
					    /* line numbers are messed up; reset them */
					    this_file->lineno = starting_points[commstuff];
					    tmp = utf_8_to_utf_16 ("<!--...");
					    add_xml_error (this_file, 903, tmp);
					    wp = xml_yytext;
					    while ((wp = uni_strchr (wp, '\n'))) {
					      wp++; this_file->lineno++;
					    }
					  }
					  xml_yytext[xml_yyleng - 1] = 0;
					  /* add this comment data to string pool */
					  xml_yylval.wstr = uni_add_string (xml_yytext);
					  return COMMDATA;
					}
<COMMSTUFF>({CommentChar}|-({CommentChar})|--+({CommentCharNoGt}))*--+> {
					  my_wchar_t *wp;
					  yy_pop_state ();
					  xml_yytext[xml_yyleng - 3] = 0;
					  /* add this comment data to string pool */
					  if ((wp = uni_utf_strstr (xml_yytext, "--")))
					    add_xml_warning (this_file, 901, uni_truncate_to (wp, 3));
					  if (xml_yyleng > 3 && xml_yytext[xml_yyleng - 4] == '-')
					    add_xml_warning (this_file, 902, utf_8_to_utf_16 ("--->"));
					  xml_yylval.wstr = uni_add_string (xml_yytext);
					  return COMMDATA;
					}

<INITIAL><!\[[\x09\x0d\x0a\x20]*(INCLUDE|IGNORE)[\x09\x0d\x0a\x20]*\[ {
					  xwrap (errdebug (7, "cond section start, line %d; level was %d; now = %d\n",
					    this_file->lineno, this_file->cond_levels, this_file->cond_levels + 1));
					  this_file->cond_levels++;
					  /* INCLUDE/IGNORE marked sections are illegal in content */
					  add_xml_error (this_file, 860, uni_truncate_to (xml_yytext, 20));
					  yy_push_state (IGNORE);
					}

<INITIAL,DTD><!\[(R?CDATA)?\[		{ my_wchar_t *wp;
					  /* unterminated marked section; skip it */
					  wp = uni_concatenate_no_free (NULL, xml_yytext);
					  wp = uni_concatenate_no_free (wp, utf_8_to_utf_16 ("..."));
					  add_xml_error (this_file, 1311, wp);
					  if (YY_START == DTD) {
					    xwrap (errdebug (7, "quasi-ignore section start, line %d; level was %d; now = %d\n",
					      this_file->lineno, this_file->cond_levels, this_file->cond_levels + 1));
					    this_file->cond_levels++;
					    /* ignore this for now */
					    starting_points[ignore] = this_file->lineno;
					    yy_push_state (IGNORE);
					  }
					}
<INITIAL,DTD><!\[r?cdata\[		{ my_wchar_t *wp;
					  /* use CDATA, not lowercase cdata */
					  add_xml_error (this_file, 368, xml_yytext);
					  wp = uni_strdup (xml_yytext);
					  uni_upcase (wp);
					  pushback (wp);
					  free (wp);
					}
<INITIAL,DTD><!\[#?PCDATA\[		{ /* no such thing as a #PCDATA marked section */
					  size_t i, j;
					  my_wchar_t *new_yytext;
					  add_xml_error (this_file, 1315, xml_yytext);
					  /* strip out P or #P and rescan */
					  new_yytext = uni_strdup (xml_yytext);
					  i = (xml_yytext[3] == '#') ? 5 : 4;
					  for (j = 3; i <= xml_yyleng; i++, j++)
					    new_yytext[j] = xml_yytext[i];
					  /* rescan as if the user wrote <![CDATA[ */
					  pushback (new_yytext);
					  free (new_yytext);
					}
<INITIAL,DTD><!\[[\x09\x0d\x0a\x20]*(R?CDATA|r?cdata)[\x09\x0d\x0a\x20]*\[ {
					  my_wchar_t *wp, *wp2, *new_delim;
					  /* isn't supposed to be whitespace around CDATA */
					  add_xml_error (this_file, 1312, xml_yytext);
					  /* strip whitespace and rescan */
					  new_delim = wp2 = uni_strdup (xml_yytext);
					  for (wp = xml_yytext; *wp != 0; wp++)
					    if (! uni_isspace (*wp))
					      *wp2++ = *wp;
					  *wp2 = 0;
					  /* rescan (previous pattern checks case (CDATA|cdata)) */
					  pushback (new_delim);
					  free (new_delim);
					}
<INITIAL,DTD><!\[R?CDATA\[({CDChar}|]{CDChar}|]]+{CDCharNoGt})*]]+> {
					  int i = 9;
					  if (YY_START != INITIAL) {
					    /* CDATA can't occur in DTD; only in text */
					    add_xml_error (this_file, 1301,
					      uni_truncate_to (xml_yytext, 20));
					  }
					  else {
					    if (xml_yytext[3] == 'R') {
					      i++;
					      /* RCDATA not allowed anywhere in XML */
					      add_xml_error (this_file, 1310, uni_truncate_to (xml_yytext, 20));
					    }
					    xml_yytext[xml_yyleng - 3] = 0;
					    /* add this CDATA string to string pool */
					    xml_yylval.wstr = uni_add_string (&xml_yytext[i]);
					    return CDATA;
					  }
					}

<INITIAL,DTD>\<!doctype			{ /* should only occur in text of top-level xml_file */
					  if (this_file->child != no)
					    /* we're in an external XML file */
					    add_xml_error (this_file, 410, xml_yytext);
					  if (YY_START == DTD)
					    /* we're already inside a DOCTYPE decl */
					    add_xml_error (this_file, 412, xml_yytext);
					  if (seen_doctype_decl)
					    /* naive users sometimes put in two DOCTYPE decls */
					    add_xml_error (this_file, 863, xml_yytext);
					  yy_push_state (DECL);
					  add_xml_error (this_file, 368, &xml_yytext[2]);
					  starting_points[decl] = this_file->lineno;
					  /* NB: DocType must match doc's root element type */
					  return DOCTYPEDECL;
					}
<INITIAL,DTD>\<!DOCTYPE			{ /* should only occur in text of top-level xml_file */
					  if (this_file->child != no)
					    /* we're in an external XML file */
					    add_xml_error (this_file, 410, xml_yytext);
					  if (YY_START == DTD)
					    /* we're already inside a DOCTYPE decl */
					    add_xml_error (this_file, 412, xml_yytext);
					  if (seen_doctype_decl)
					    /* naive users sometimes put in two DOCTYPE decls */
					    add_xml_error (this_file, 863, xml_yytext);
					  yy_push_state (DECL);
					  starting_points[decl] = this_file->lineno;
					  /* NB: DocType must match doc's root element type */
					  return DOCTYPEDECL;
					}
<DECL>\[				{ /* switch input streams if there was a pub or sys
					   * identifier; switch back when we hit <<EOF>>
					   */
					  yy_push_state (DTD);
					  starting_points[dtd] = this_file->lineno;
					  return BEGIN_DTD_DELIM;
					}


<DTD><ELEMENT				{ add_xml_error (this_file, 673, xml_yytext);
					  yy_push_state (DECL);
					  /* turn on balanced-parend checking for perefs */
					  expecting_cpseq_or_choice = 1;
					  starting_points[decl] = this_file->lineno;
					  return ELEMENTDECL_DELIM;
					}
<DTD><!element				{ yy_push_state (DECL);
					  /* turn on balanced-parend checking for perefs */
					  expecting_cpseq_or_choice = 1;
					  add_xml_error (this_file, 368, &xml_yytext[2]);
					  starting_points[decl] = this_file->lineno;
					  return ELEMENTDECL_DELIM;
					}
<DTD><!ELEMENT				{ yy_push_state (DECL);
					  /* turn on balanced-parend checking for perefs */
					  expecting_cpseq_or_choice = 1;
					  starting_points[decl] = this_file->lineno;
					  return ELEMENTDECL_DELIM;
					}
<DTD><ATTLIST				{ add_xml_error (this_file, 591, xml_yytext);
					  starting_points[decl] = this_file->lineno;
					  yy_push_state (DECL);
					  return ATTLISTDECL_DELIM;
					}
<DTD><!attlist				{ /* Note special attribute vals, xml:spaces & xml:lang */
					  yy_push_state (DECL);
					  add_xml_error (this_file, 368, &xml_yytext[2]);
					  starting_points[decl] = this_file->lineno;
					  /* Emit warning if element isn't declared elsewhere */
					  return ATTLISTDECL_DELIM;
					}
<DTD><!ATTLIST				{ /* Note special attribute vals, xml:spaces & xml:lang */
					  yy_push_state (DECL);
					  starting_points[decl] = this_file->lineno;
					  /* Emit warning if element isn't declared elsewhere */
					  return ATTLISTDECL_DELIM;
					}
<DTD><ENTITY[\x09\x0d\x0a\x20]+%	{ add_xml_error (this_file, 752, xml_yytext);
					  starting_points[decl] = this_file->lineno;
					  yy_push_state (DECL);
					  return PARAMETER_ENTITY_DECL_DELIM;
					}
<DTD><!ENTITY[\x09\x0d\x0a\x20]+%	{ yy_push_state (DECL);
					  starting_points[decl] = this_file->lineno;
					  return PARAMETER_ENTITY_DECL_DELIM;
					}
<DTD><ENTITY				{ add_xml_error (this_file, 802, xml_yytext);
					  starting_points[decl] = this_file->lineno;
					  yy_push_state (DECL);
					  return ENTITYDECL_DELIM;
					}
<DTD><!entity				{ yy_push_state (DECL);
					  add_xml_error (this_file, 368, &xml_yytext[2]);
					  starting_points[decl] = this_file->lineno;
					  return ENTITYDECL_DELIM;
					}
<DTD><!ENTITY				{ yy_push_state (DECL);
					  starting_points[decl] = this_file->lineno;
					  return ENTITYDECL_DELIM;
					}
<DTD><NOTATION				{ add_xml_error (this_file, 703, xml_yytext);
					  starting_points[decl] = this_file->lineno;
					  yy_push_state (DECL);
					  return NOTATIONDECL_DELIM;
					}
<DTD><!notation				{ yy_push_state (DECL);
					  add_xml_error (this_file, 368, &xml_yytext[2]);
					  starting_points[decl] = this_file->lineno;
					  return NOTATIONDECL_DELIM;
					}
<DTD><!NOTATION				{ yy_push_state (DECL);
					  starting_points[decl] = this_file->lineno;
					  return NOTATIONDECL_DELIM;
					}
<DTD><\/?(({Letter})|_|:)({NameChar})*([\"\'][^\"\']*[\'\"]|[^>])*> {
					  int errnum;
					  /* oops; <markup> appears in DTD or ext PEnt */
					  errnum = validating (this_file) ? 464 : 465;
					  add_xml_error (this_file, errnum, xml_yytext);
					}

<DTD><!\[[\x09\x0d\x0a\x20]*%(({Letter})|_|:)({NameChar})*;([\x09\x0d\x0a\x20]*\[)? {
					  my_wchar_t *expansion, *tmp;
					  if (! in_external_dtd_subset (this_file) && YY_START == DECL) {
					    /* PErefs can only occur inside of markup in ext subset */
					    add_xml_error (this_file, 1000,
					      uni_truncate_to (xml_yytext, 20));
					  }
					  if (! validating (this_file)) {
					    xwrap (errdebug (7, "cond section start, line %d; level was %d; now = %d\n",
					      this_file->lineno, this_file->cond_levels, this_file->cond_levels + 1));
					    this_file->cond_levels++;
					    /* ignore this for now */
					    starting_points[ignore] = this_file->lineno;
					    yy_push_state (IGNORE);
					  }
					  else {
					    /* recursively expand entity; don't pad here */
					    expansion = map_entities (this_file, xml_yytext,
					      MAP_PARAMETER_ENTITIES | ABORT_ON_FAILURE, 0);
					    if (expansion == NULL)
					      add_xml_error (this_file, 450, 
						uni_truncate_to (xml_yytext, 20));
					    else {
					      uni_utf_strip (expansion, "\x09\x0d\x0a\x20");
					      if (uni_utf_strncmp (expansion, "<![IGNORE[", 10) == 0) {
						xwrap (errdebug (7, "IGNORE section start, line %d; level was %d; now = %d\n",
						  this_file->lineno, this_file->cond_levels, this_file->cond_levels + 1));
						this_file->cond_levels++;
						xwrap (errdebug (5, "<![ + peref + [ maps to <![IGNORE["));
						if (! in_external_dtd_subset (this_file))
						  /* Dunno why INCLUDE/IGNORE aren't OK in internal subset */
						  add_xml_error (this_file, 861, expansion);
						starting_points[ignore] = this_file->lineno;
						yy_push_state (IGNORE);
					      }
					      else if (uni_utf_strncmp (expansion, "<![INCLUDE[", 11) == 0) {
						xwrap (errdebug (7, "INCLUDE section start, line %d; level was %d; now = %d\n",
						  this_file->lineno, this_file->cond_levels, this_file->cond_levels + 1));
						this_file->cond_levels++;
						xwrap (errdebug (5, "<![ + peref + [ maps to <![INCLUDE["));
						if (! in_external_dtd_subset (this_file))
						  /* Dunno why INCLUDE/IGNORE aren't OK in internal subset */
						  add_xml_error (this_file, 861, expansion);
						starting_points[include] = this_file->lineno;
						yy_push_state (YY_START);
						if (uni_strlen (expansion) > 11)
						  push_xml_string (this_file, &expansion[11]);
					      }
					      else {
						/* rescan expansion */
						xwrap (errdebug (5, "<![+peref+[ != <![INCLUDE[/<![IGNORE"));
						if (YY_START == DTD && validating (this_file)) {
						  tmp = malloc (50 * sizeof (my_wchar_t));
						  uni_strcpy (tmp, uni_truncate_to (xml_yytext, 20));
						  uni_utf_strcat (tmp, " (-> ");
						  uni_strcat (tmp, uni_truncate_to (expansion, 20));
						  uni_utf_strcat (tmp, ")");
						  add_xml_error (this_file, 862, tmp);
						  free (tmp);
						}
						if (uni_strlen (expansion) <= xml_yyleng)
						  pushback (expansion);
						else
						  /* necessitated here by an apparent Flex bug */
						  push_xml_string (this_file, expansion);
					      }
					      /* map_entities() returns a malloc'd string */
					      free (expansion);
					    }
					  }
					}
<DTD><!\[[\x09\x0d\x0a\x20]*INCLUDE[\x09\x0d\x0a\x20]*\[ {
					  xwrap (errdebug (7, "INCLUDE section start, line %d; level was %d; now = %d\n",
					    this_file->lineno, this_file->cond_levels, this_file->cond_levels + 1));
					  this_file->cond_levels++;
					  if (! in_external_dtd_subset (this_file))
					    /* Dunno why INCLUDE/IGNORE are not OK in internal subset. */
					    add_xml_error (this_file, 861,
					      uni_truncate_to (xml_yytext, 14));
					  starting_points[include] = this_file->lineno;
					  yy_push_state (YY_START);
					}
<DTD><!\[[\x09\x0d\x0a\x20]*INCLUDE	{ xwrap (errdebug (7, "INCLUDE section start, line %d; level was %d; now = %d\n",
					    this_file->lineno, this_file->cond_levels, this_file->cond_levels + 1));
					  this_file->cond_levels++;
					  if (! in_external_dtd_subset (this_file))
					    /* Dunno why INCLUDE/IGNORE are not OK in internal subset. */
					    add_xml_error (this_file, 861,
					      uni_truncate_to (xml_yytext, 14));
					  add_xml_error (this_file, 852,
					    uni_truncate_to (xml_yytext, 20));
					  starting_points[include] = this_file->lineno;
					  yy_push_state (YY_START);
					}
<DTD><!\[[\x09\x0d\x0a\x20]*IGNORE[\x09\x0d\x0a\x20]*\[ {
					  xwrap (errdebug (7, "IGNORE section start, line %d; level was %d; now = %d\n",
					    this_file->lineno, this_file->cond_levels, this_file->cond_levels + 1));
					  this_file->cond_levels++;
					  if (! in_external_dtd_subset (this_file))
					    /* Dunno why INCLUDE/IGNORE are not OK in internal subset. */
					    add_xml_error (this_file, 861,
					      uni_truncate_to (xml_yytext, 14));
					  starting_points[ignore] = this_file->lineno;
					  yy_push_state (IGNORE);
					}
<DTD><!\[[\x09\x0d\x0a\x20]*IGNORE	{ xwrap (errdebug (7, "IGNORE section start, line %d; level was %d; now = %d\n",
					    this_file->lineno, this_file->cond_levels, this_file->cond_levels + 1));
					  this_file->cond_levels++;
					  if (! in_external_dtd_subset (this_file))
					    /* Dunno why INCLUDE/IGNORE are not OK in internal subset. */
					    add_xml_error (this_file, 861,
					      uni_truncate_to (xml_yytext, 14));
					  add_xml_error (this_file, 852,
					    uni_truncate_to (xml_yytext, 20));
					  starting_points[ignore] = this_file->lineno;
					  yy_push_state (IGNORE);
					}
<DTD,IGNORE>]]>				{ xwrap (errdebug (7, "cond section end, line %d; level was %d; now = %d\n",
					    this_file->lineno, this_file->cond_levels, this_file->cond_levels - 1));
					  if (--this_file->cond_levels >= 0)
					    yy_pop_state ();
					  else {
					    this_file->cond_levels = 0;
					    add_xml_error (this_file, 851, xml_yytext);
					  }
					}
<IGNORE><\?(({Letter})|_|:)({NameChar})*({PiChar}|\?{CharNoGt})+\?> {
					  xwrap (errdebug (7, "bypassed PI ending at line %d\n", this_file->lineno));
					}
<IGNORE><!--({CommentChar}|-{CommentChar}|--+{CommentCharNoGt})*--+> {
					  xwrap (errdebug (7, "bypassed comment ending at line %d\n", this_file->lineno));
					}
<IGNORE,IGNORE_MARKED_SECT><!\[[\x09\x0d\x0a\x20]*(%|IGNORE|INCLUDE) {
					  xwrap (errdebug (7, "bypassing <[!... starting at line %d\n", this_file->lineno));
					  starting_points[marked] = this_file->lineno;
					  yy_push_state (IGNORE_MARKED_SECT);
					}
<IGNORE_MARKED_SECT>]]>			{ xwrap (errdebug (7, "bypassed ...]]> ending at line %d\n", this_file->lineno));
					  yy_pop_state ();
					}
<IGNORE><				{ xwrap (errdebug (7, "bypassing markup beginning at line %d\n", this_file->lineno));
					  yy_push_state (IGNORE_DECL);
					}
<IGNORE_DECL>>				{ xwrap (errdebug (7, "done bypassing markup, line %d\n", this_file->lineno));
					  yy_pop_state ();
					}
<IGNORE_DECL>(\"[^\"]*\")|(\'[^\']*\')		{ /* do nothing */ }
<IGNORE>[\x09\x0d\x0a\x20]			{ /* do nothing */ }
<IGNORE,IGNORE_DECL,IGNORE_MARKED_SECT>.|\n	{ /* do nothing */ }

<DTD><[\x09\x0d\x0a\x20]+!		{ my_wchar_t *wp;
					  xml_yytext[1] = 0;
					  /* spurious whitespace after < in "< !" */
					  add_xml_error (this_file, 1114, xml_yytext);
					  wp = uni_strdup (utf_8_to_utf_16 ("<!"));
					  pushback (wp);
					  free (wp);
					}
<DTD><![\x09\x0d\x0a\x20]+		{ my_wchar_t *wp;
					  add_xml_error (this_file, 502,
					    uni_truncate_to (xml_yytext, 20));
					  xml_yytext[2] = 0;
					  wp = uni_strdup (xml_yytext);
					  pushback (wp);
					  free (wp);
					}
<DTD><![^[%>\x09\x0d\x0a\x20\-]*	{ xml_yylval.wstr = uni_add_string (&xml_yytext[2]);
					  yy_push_state (DECL);
					  return BAD_DECL;
					}

<DTD,DECL>&(({Letter})|_|:)({NameChar})*; {
					  /* erefs illegal here, except in ent or attvals */
					  add_xml_error (this_file, 460,
					    uni_truncate_to (xml_yytext, 20));
					}
<DTD,DECL>&(#({Digit})+|#x[0-9a-fA-F]+); {
					  /* char refs illegal, too; see the previous pattern */
					  add_xml_error (this_file, 461,
					    uni_truncate_to (xml_yytext, 20));
					}
<DTD,DECL>%(({Letter})|_|:)({NameChar})*; {
					  my_wchar_t *entname, *exp1, *exp2;
					  if (! in_external_dtd_subset (this_file) && YY_START == DECL) {
					    /* PErefs can only occur inside of markup in ext subset */
					    add_xml_error (this_file, 1000,
					      uni_truncate_to (xml_yytext, 20));
					  }
					  /* recursively expand entity */
					  xml_yytext[xml_yyleng - 1] = 0;
					  entname = &xml_yytext[1];
					  exp1 = expand_peref (this_file, entname, WITH_WHITESPACE);
					  if (exp1 == NULL)
					    add_xml_error (this_file, 450,
					      uni_truncate_to (entname, 20));
					  else {
					    exp1 = uni_strdup (exp1);
					    if (validating (this_file)) {
					      exp2 = map_entities (this_file, exp1,
					        UNRESOLVABLES_OKAY |
						MAP_PARAMETER_ENTITIES |
					        PAD_PARAMETER_ENTITIES, 0);
					      if (exp2) {
					        if (check_gt_and_lt (exp2))
						  /* make sure replacement text is <balanced> */
						  add_xml_error (this_file, 775, entname);
					        if (expecting_cpseq_or_choice)
						  /* if in a content model, check for balanced parends */
						  check_parend_nesting (entname, exp2);
						free (exp2);
					      }
					      /* switch input buffer to string and rescan */
					      push_xml_string (this_file, exp1);
					      free (exp1);
					    }
					  }
					}
<DTD>%[^%\]<>;\x09\x0d\x0a\x20]*	{ /* unterminated PE reference */
					  add_xml_error (this_file, 451,
					    uni_truncate_to (&xml_yytext[1], 20));
					}

<DTD>]					{ if (in_external_dtd_subset (this_file))
					    add_xml_error (this_file, 411, xml_yytext);
					  else
					    /* [] is okay around the _internal_ DTD subset */
					    yy_pop_state ();
					  return END_DTD_DELIM;
					}

<DTD,DECL,MARKUP><			{ if (YY_START != DTD)
					    add_xml_error (this_file, 505, xml_yytext);
					  add_xml_error (this_file, 504, xml_yytext);
					}
<DECL,MARKUP>--({CommentChar}|-{CommentChar})*-- {
					  /* apparent old-style SGML comment */
					  xml_yytext[xml_yyleng - 2] = 0;
					  add_xml_error (this_file, 900, &xml_yytext[2]);
					}
<DECL,XMLDECL,MARKUP>=			{ return EQUALS; }
<DECL>[\x09\x0d\x0a\x20]-\ [\-Oo][\x09\x0d\x0a\x20] {
					  add_xml_error (this_file, 655, xml_yytext);
					}
<DECL>\(				{ return LEFTPAREND; }
<DECL>\)				{ return RIGHTPAREND; }
<DECL>\|				{ return VSLASH; }
<DECL>\?				{ return QUESTIONMARK; }
<DECL>\*				{ return STAR; }
<DECL>\+				{ return PLUS; }
<DECL>\-				{ return MINUS; }
<DECL>,					{ return COMMA; }
<DECL>&					{ return AMPERSAND; }

<DECL>#[Aa][Ll][Ll]			{ /* add this Nmtoken to string pool */
					  add_xml_error (this_file, 685, xml_yytext);
					  xml_yylval.wstr = uni_add_string (xml_yytext);
					  return NMTOKEN;
					}
<DECL>#[Ff][Ii][Xx][Ee][Dd]		{ /* We accept upper or lowercase here; but in the
					   * parser the case gets checked, and an error msg
					   * gets issued if lowercase letters are found.
					   */
					  xml_yylval.wstr = uni_add_string (xml_yytext);
					  return FIXED;
					}
<DECL>#[Rr][Ee][Qq][Uu][Ii][Rr][Ee][Dd] {
					  /* see comments above */
					  xml_yylval.wstr = uni_add_string (xml_yytext);
					  return REQUIRED;
					}
<DECL>#[Ii][Mm][Pp][Ll][Ii][Ee][Dd]	{ /* see comments above */
					  xml_yylval.wstr = uni_add_string (xml_yytext);
					  return IMPLIED;
					}
<DECL>#DEFAULT				{ xml_yylval.wstr = uni_add_string (xml_yytext);
					  return DEFAULT;
					}
<DECL>#P?CDATA				{ if (xml_yytext[1] != 'P')
					     /* sometimes people accidentally write #CDATA */
					     add_xml_error (this_file, 684, xml_yytext);
					   xml_yylval.wstr = uni_add_string (xml_yytext);
					   return PCDATA;
					}

<DECL>[Pp][Uu][Bb][Ll][Ii][Cc]		{ /* We accept upper or lowercase here; but in the
                                           * parser the case gets checked, and an error msg
                                           * gets issued if lowercase letters are found.
                                           * An error is also issued for a missing '#' char
                                           * for FIXED|REQUIRED|IMPLIED_NOPOUND.
                                           */
					  xml_yylval.wstr = uni_add_string (xml_yytext);
					  return PUBLIC;
					}
<DECL>[Ss][Yy][Ss][Tt][Ee][Mm]		{ /* see comments above */
					  xml_yylval.wstr = uni_add_string (xml_yytext);
					  return SYSTEM;
					}
<DECL>#?[Ee][Mm][Pp][Tt][Yy]		{ if (*xml_yytext == '#') {
					    /* Oops; drop the '#' from #EMPTY */
					    add_xml_error (this_file, 1112, xml_yytext);
					    xml_yytext++;
					  }
					  /* see comments above */
					  xml_yylval.wstr = uni_add_string (xml_yytext);
					  return EMPTY; 
					}					
<DECL>[Aa][Nn][Yy]			{ /* see comments above */
					  xml_yylval.wstr = uni_add_string (xml_yytext);
					  return ANY;
					}
<DECL>[Rr][Cc][Dd][Aa][Tt][Aa]		{ /* see comments above */
					  xml_yylval.wstr = uni_add_string (xml_yytext);
					  return RCDATA; 
					}
<DECL>[Cc][Dd][Aa][Tt][Aa]		{ /* see comments above */
					  xml_yylval.wstr = uni_add_string (xml_yytext);
					  return CDATA_; 
					}
<DECL>[Nn][Aa][Mm][Ee][Ss]?		{ /* see comments above */
					  xml_yylval.wstr = uni_add_string (xml_yytext);
					  return NAMES; 
					}
<DECL>[Nn][Uu][Mm][Bb][Ee][Rr][Ss]?	{ /* see comments above */
					  xml_yylval.wstr = uni_add_string (xml_yytext);
					  return NUMBERS; 
					}
<DECL>[Nn][Uu][Tt][Oo][Kk][Ee][Nn][Ss]?	{ /* see comments above */
					  xml_yylval.wstr = uni_add_string (xml_yytext);
					  return NUTOKENS; 
					}
<DECL>[Ii][Dd]				{ /* see comments above */
					  xml_yylval.wstr = uni_add_string (xml_yytext);
					  return ID;
					}
<DECL>[Ii][Dd][Rr][Ee][Ff]		{ /* see comments above */
					  xml_yylval.wstr = uni_add_string (xml_yytext);
					  return IDREF;
					}
<DECL>[Ee][Nn][Tt][Ii][Tt][Yy]		{ /* see comments above */
					  xml_yylval.wstr = uni_add_string (xml_yytext);
					  return ENTITY;
					}
<DECL>[Nn][Mm][Tt][Oo][Kk][Ee][Nn]	{ /* see comments above */
					  xml_yylval.wstr = uni_add_string (xml_yytext);
					  /* NMTOKEN_ is the actual keyword; cf. NMTOKEN */
					  return NMTOKEN_;
					}
<DECL>[Ii][Dd][Rr][Ee][Ff][Ss]		{ /* see comments above */
					  xml_yylval.wstr = uni_add_string (xml_yytext);
					  return IDREFS;
					}
<DECL>[Ee][Nn][Tt][Ii][Tt][Ii][Ee][Ss]	{ /* see comments above */
					  xml_yylval.wstr = uni_add_string (xml_yytext);
					  return ENTITIES;
					}
<DECL>[Nn][Mm][Tt][Oo][Kk][Ee][Nn][Ss]	{ /* see comments above */
					  xml_yylval.wstr = uni_add_string (xml_yytext);
					  return NMTOKENS;
					}
<DECL>[Nn][Oo][Tt][Aa][Tt][Ii][Oo][Nn]	{ /* see comments above */
					  xml_yylval.wstr = uni_add_string (xml_yytext);
					  return NOTATION;
					}
<DECL>[Nn][Dd][Aa][Tt][Aa]		{ /* see comments above */
					  xml_yylval.wstr = uni_add_string (xml_yytext);
					  return NDATA;
					}
<DECL>[Ss][Dd][Aa][Tt][Aa]		{ /* see comments above */
					  xml_yylval.wstr = uni_add_string (xml_yytext);
					  return SDATA;
					}

<DECL,XMLDECL,MARKUP>(\"[^"]*\")|(\'[^']*\') {
					 my_wchar_t c;
					 size_t i, len;
					  /* '" beginning quote, will match ending one */
					  len = xml_yyleng - 1;
					  xml_yytext[len] = 0;
					  /* we'll deal with reference expansion in the parser */
					  xml_yylval.wstr = uni_add_string (&xml_yytext[1]);
					  for (i = 1; i < len; i++) {
					    c = xml_yytext[i];
					    if (c >= 0xD800 && c <= 0xDFFF) /* surrogate block? */
					       add_xml_error (this_file, 1054, xml_yylval.wstr);
					    else if (c <= 0x1F && ! (c == 0x09 || c == 0x0A || c == 0x0D))
					       add_xml_error (this_file, 1056, xml_yylval.wstr);
					    else if (c == 0xFFFE) /* byte-order mark? */
					       add_xml_error (this_file, 1055, xml_yylval.wstr);
					    else if (c == 0xFFFF) /* non-Unicode? */
					       add_xml_error (this_file, 1057, xml_yylval.wstr);
					    else if (c >= 0x80 && c <= 0x9F) {
					       /* A "C1" (as opposed to ASCII "C0") control character */
					       add_xml_warning (this_file, 1058, xml_yylval.wstr);
					     }
					  }
					  return QUOTEDSTRING;
					}
<DECL,XMLDECL,MARKUP>['"]([^"'<>\/]|\/[^'"<>])*(['"<>]|\/>) {
					  size_t i; my_wchar_t *tmp;
					  /* '" or nearest '>' or "/>" ends malformed quotation */
					  add_xml_error (this_file, 510, xml_yytext);
					  /* strip out trailing metacharacters */
					  for (i = xml_yyleng - 1; i > 0; i--)
					    if (! uni_utf_any (&xml_yytext[i], "'\"<>/"))
					      break;
					  tmp = uni_strdup (&xml_yytext[i + 1]);
					  xml_yytext[i + 1] = 0;
					  /* deal with reference expansion in the parser */
					  xml_yylval.wstr = uni_add_string (&xml_yytext[1]);
					  pushback (tmp);
					  free (tmp);
					  return QUOTEDSTRING;
					}

<DECL,XMLDECL,MARKUP>(({Letter})|_|:)({NameChar})* {
					  /* add this Name to string pool */
					  xml_yylval.wstr = uni_add_string (xml_yytext);
					  return NAME;
					}
<DECL>({NameChar})+			{ /* add this Nmtoken to string pool */
					  xml_yylval.wstr = uni_add_string (xml_yytext);
					  /* cf. NMTOKEN_ above, which is the "NMTOKEN" keyword */
					  return NMTOKEN;
					}
<DTD>[^<\"&\'%\]\x09\x0d\x0a\x20]+	{ xml_yylval.wstr = uni_add_string (xml_yytext);
					  xwrap (errdebug (7, "ran into junk string, %s, in state %d\n",
					    utf_16_to_utf_8 (xml_yytext), YY_START));
					  switch (YY_START) {
					  case DTD:
					    add_unique_error (this_file, 466, uni_truncate_to (xml_yytext, 50));
					    break;
					  default:
					    add_xml_error (this_file, 462, uni_truncate_to (xml_yytext, 50));
					    break;
					  }
					  /* skip it; don't tell the parser about it */
					}
<DECL,DTD,MARKUP,XMLDECL>[\x09\x0d\x0a\x20] {
					  /* do nothing */
					}
<DECL>>					{ yy_pop_state ();
					  /* turn off check on balanced PE parends */
					  expecting_cpseq_or_choice = 0;
					  return DECL_END;
					}
<DECL,XMLDECL,MARKUP>[!@#$%^&*()_+\-{}[\]:;|\\,.?\/~`] {
					  if (++bad_char_count > 100) {
					    add_xml_error (this_file, 1352, xml_yytext);
					    while (pop_xml_file ());
					    BEGIN (0); yyterminate ();
					  }
					  /* unused punctuation inside of markup */
					  add_xml_error (this_file, 1122, xml_yytext);
					} 

<*>({Control})				{ char tmp[32];
					  if (++bad_char_count > 100) {
					    add_xml_error (this_file, 1352, xml_yytext);
					    while (pop_xml_file ());
					    BEGIN (0); yyterminate ();
					  }
					  /* control characters are generally illegal */
					  sprintf (tmp, "&#x%x;", (unsigned int)xml_yytext[0]);
					  add_xml_error (this_file, 1354, utf_8_to_utf_16 (tmp));
					}
<*>.|\n					{ if (++bad_char_count > 100) {
					    add_xml_error (this_file, 1352, xml_yytext);
					    while (pop_xml_file ());
					    BEGIN (0); yyterminate ();
					  }
					  xml_yylval.wstr = uni_add_string (xml_yytext);
					  xwrap (errdebug (7, "ran into bad char %d (%s), in state %d\n",
					    xml_yytext[0], int_2_utf_8_string (xml_yytext[0]), YY_START));
					  return BAD_CHAR;
					}

<<EOF>>					{ my_wchar_t *tmp;
					  char intbuf[32];
					  enum yesno at_top_level = no;
					  /* eof_count is used to prevent looping */
					  eof_count++;
					  xwrap (xwrap (errdebug (5, "Matched <<EOF>> in lexer.\n")));
					  switch (what_are_we_scanning ())
					    {
					    case string:
					      pop_xml_file ();
					      xwrap (errdebug (7, "Lexer: Done with string object.\n"));
					      /* on EOF unmark all starting points */
					      memset (starting_points, 0, STATECOUNT * sizeof (int));
					      break;
					    case entity_replacement_text:
					      xwrap (errdebug (7, "Done w/ ent text; return END_FILE.\n"));
					      /* Hmmm; are we looping somehow? */
					      if (eof_count > 5000)
						if (xml_file_stack_len > 1)
						  /* if there's room, try popping a file off the stack */
						  pop_xml_file ();
						else {
						  eof_count = bad_char_count = 0;
						  while (pop_xml_file ());
						  /* if not, terminate processing */
						  BEGIN (0); yyterminate ();
						}
					      /* on EOF unmark all starting points */
					      memset (starting_points, 0, STATECOUNT * sizeof (int));
					      /* the parser usually is the one to call pop_xml_file () */
					      return END_FILE;
					      break;
					    case file:
					      if (xml_file_stack_len <= 1)
						at_top_level = yes;
					      /* end of parent xml_file */
					      switch (YY_START)
						{
						case MARKUP:
						  sprintf (intbuf, "%d", starting_points[markup]);
						  tmp = utf_8_to_utf_16 (intbuf);
						  add_xml_error (this_file, 1120, tmp);
						  break;
						case XMLDECL:
						  sprintf (intbuf, "%d", starting_points[xmldecl]);
						  tmp = utf_8_to_utf_16 (intbuf);
						  add_xml_error (this_file, 371, tmp);
						  break;
						case PISTUFF:
						  sprintf (intbuf, "%d", starting_points[pistuff]);
						  tmp = utf_8_to_utf_16 (intbuf);
						  add_xml_error (this_file, 372, tmp);
						  break;
						case DECL:
						  sprintf (intbuf, "%d", starting_points[decl]);
						  tmp = utf_8_to_utf_16 (intbuf);
						  add_xml_error (this_file, 503, tmp);
						  break;
						case DTD:
						  if (at_top_level == yes) {
						    /* an error if we're not in an external entity */
						    sprintf (intbuf, "%d", starting_points[dtd]);
						    tmp = utf_8_to_utf_16 (intbuf);
						    add_xml_error (this_file, 452, tmp);
						  }
						  /* fall through (whether at top level or not) */
						case INITIAL:
						  if (this_file->cond_levels > 0) {
						    tmp = utf_8_to_utf_16 ("EOF");
						    add_xml_error (this_file, 850, tmp);
						  }
						  break;
						case IGNORE:
						case IGNORE_DECL:
						  sprintf (intbuf, "%d", starting_points[ignore]);
						  tmp = utf_8_to_utf_16 (intbuf);
						  add_xml_error (this_file, 1120, tmp);
						  break;
						case IGNORE_MARKED_SECT:
						  sprintf (intbuf, "%d", starting_points[marked]);
						  tmp = utf_8_to_utf_16 (intbuf);
						  add_xml_error (this_file, 1120, tmp);
						  break;
						}
					      if (at_top_level == yes) {
						while (pop_xml_file ());
						eof_count = bad_char_count = 0;
						/* on EOF unmark all starting points */
						memset (starting_points, 0, STATECOUNT * sizeof (int));
						xwrap (errdebug (7, "Lexer done; calling yyterminate().\n"));
						BEGIN (0); yyterminate ();
					      }
					      else {
						xwrap (errdebug (7, "Done w/ file obj; return END_FILE.\n"));
					        /* Hmmm; are we looping somehow? */
					        if (eof_count > 5000)
						  /* if so, try popping a file off the input stack */
						  pop_xml_file ();
						/* on EOF unmark all starting points */
						memset (starting_points, 0, STATECOUNT * sizeof (int));
						/* the parser usually is the one to call pop_xml_file () */
						return END_FILE;
					      }
					    break;
					  }
					}

%%


/*
 * what_are_we_scanning
 *
 *   Returns the type of object on top of the xml_file_stack.  The
 *   possibilities are file, entity_replacement_text, or string.
 *
 *   This routine is used by the <<EOF>> target and the pop_xml_file()
 *   routine.  At <<EOF>>, if we are scanning a string object, then an
 *   END_FILE token should not be returned.  In pop_xml_file(), if we
 *   are scanning a string, we should not pop any lexer states.  See
 *   <<EOF>> just above, and pop_xml_file() below.
 */
enum scan_objects
what_are_we_scanning (void)
{

  /* If we're still in the top-level xml_file, return file */
  if (xml_file_stack_len == 0)
    return file;

  /* But if we're in a child, see if we're currently scanning
   * a string, or whether we're reading directly from a file.
   */
  return xml_file_stack[xml_file_stack_len - 1].what_object;

}


/*
 * push_xml_file
 *
 *   Pushes another scan object (in this case an xml_file) onto the
 *   xml_file_stack; pushes two new states onto the lexer state stack,
 *   too:  1) a lexer stack marker, and 2) an arbitrary new lexer
 *   state (argument 2).  Returns the new xml_file_stack_len.
 *
 *   The lexer stack marker is there so that when we pop this object
 *   off the xml_file_stack, we know (even if there are errors) where
 *   to stop popping off associated lexer states (see arg 2).
 *   
 */
int
push_xml_file (struct xml_file *xf, int lexer_state)
{

  size_t size;

  if (++xml_file_stack_len > xml_file_stack_buflen)
    {
      size = sizeof (struct xml_file_state);
      if (xml_file_stack == NULL)
	{
	  xml_file_stack_buflen = 1;
	  if ((xml_file_stack = malloc (size * xml_file_stack_buflen)) == NULL)
	    errabort (40, "malloc() error in %s\n", "push_xml_file()");
	}
      else
	{
	  xml_file_stack_buflen = xml_file_stack_len;
	  if ((xml_file_stack = realloc (xml_file_stack, size * xml_file_stack_buflen)) == NULL)
	    errabort (41, "realloc() error in %s\n", "push_xml_file()");
	}
    }

  if (xml_file_stack_len == 1)
    {
      xml_file_stack[xml_file_stack_len - 1].buffer = YY_CURRENT_BUFFER;
      xml_file_stack[xml_file_stack_len - 1].what_object = file;
      xml_file_stack[xml_file_stack_len - 1].xf = this_file;
      xml_file_stack[xml_file_stack_len - 1].lexer_stack_marker = 0;
      return push_xml_file (xf, lexer_state);
    }

  /* have flex read input from the child xml_file */
  xml_file_stack[xml_file_stack_len - 1].buffer = xml_yy_create_buffer (NULL, 16384);
  xml_yy_switch_to_buffer (xml_file_stack[xml_file_stack_len - 1].buffer);
  xml_file_stack[xml_file_stack_len - 1].what_object = file;
  xml_file_stack[xml_file_stack_len - 1].xf = xf;
  /* when done with this object, we'll pop lexer states 'til we get to this marker */
  xml_file_stack[xml_file_stack_len - 1].lexer_stack_marker = MARKER_BASE + xml_file_stack_len;
  this_file = xf;

  rewind_xml_file (xf);
  yy_push_state (xml_file_stack[xml_file_stack_len - 1].lexer_stack_marker);
  yy_push_state (lexer_state);

  xwrap (errdebug (7, "Pushed XML file; now scanning it: %s\n", this_file->filename));
  xwrap (errdebug (7, "Lexer object stack len = %d\n", xml_file_stack_len));
  xwrap (errdebug (7, "Lexer state now = %d (pushed marker, %d, and new state, %d)\n",
    yy_top_state(), xml_file_stack[xml_file_stack_len - 1].lexer_stack_marker, lexer_state));
  return xml_file_stack_len;

}


/*
 * push_entity_text
 *
 *   Pushes another scan object (in this case the replacement text for
 *   some entity declared in a DTD) onto the xml_file_stack; pushes
 *   two new states onto the lexer state stack, too: 1) a lexer stack
 *   marker, and 2) an arbitrary new lexer state (argument 2).
 *   Returns the new xml_file_stack_len.
 *
 *   The lexer stack marker is there so that when we pop this object
 *   off the xml_file_stack, we know (even if there are errors) where
 *   to stop popping off associated lexer states (see arg 2).  Note
 *   that xml_file_stack is something of a misnomer.  In fact, it's
 *   possible to push entity replacement text as well.
 *
 *   The parser uses this routine to temporarily push entity replacement
 *   text so it can be given a cursory check.  Note that to check it, we
 *   need to be in the INITIAL state (so we can recognize entity refs
 *   and such).  So push_entity_text() should normally be called with a
 *   zero lexer_state argument.
 */
int
push_entity_text (struct xml_file *xf, my_wchar_t *wp, int lexer_state)
{

  size_t size;

  if (++xml_file_stack_len > xml_file_stack_buflen)
    {
      size = sizeof (struct xml_file_state);
      if (xml_file_stack == NULL)
	{
	  xml_file_stack_buflen = 1;
	  if ((xml_file_stack = malloc (size * xml_file_stack_buflen)) == NULL)
	    errabort (40, "malloc() error in %s\n", "push_xml_file()");
	}
      else
	{
	  xml_file_stack_buflen = xml_file_stack_len;
	  if ((xml_file_stack = realloc (xml_file_stack, size * xml_file_stack_buflen)) == NULL)
	    errabort (41, "realloc() error in %s\n", "push_xml_file()");
	}
    }

  if (xml_file_stack_len == 1)
    {
      xml_file_stack[xml_file_stack_len - 1].buffer = YY_CURRENT_BUFFER;
      xml_file_stack[xml_file_stack_len - 1].what_object = file;
      xml_file_stack[xml_file_stack_len - 1].xf = this_file;
      xml_file_stack[xml_file_stack_len - 1].lexer_stack_marker = 0;
      return push_entity_text (xf, wp, lexer_state);
    }

  /* have flex read input from the child xml_file */
  xml_file_stack[xml_file_stack_len - 1].buffer = yy_scan_string (wp);
  /* no need to call xml_yy_switch_to_buffer () */
  xml_file_stack[xml_file_stack_len - 1].what_object = entity_replacement_text;
  xml_file_stack[xml_file_stack_len - 1].xf = xf;
  /* when done with this object, we'll pop lexer states 'til we get to this marker */
  xml_file_stack[xml_file_stack_len - 1].lexer_stack_marker = MARKER_BASE + xml_file_stack_len;
  this_file = xf;

  /* push current state (so when pop_xml_file pops it, nothing changes) */
  yy_push_state (xml_file_stack[xml_file_stack_len - 1].lexer_stack_marker);
  yy_push_state (lexer_state);

  xwrap (errdebug (7, "Pushed XML entity replacment text; now scanning it: \"%s\"\n", utf_16_to_utf_8 (wp)));
  xwrap (errdebug (7, "Lexer object stack len = %d\n", xml_file_stack_len));
  xwrap (errdebug (7, "Lexer state now = %d (pushed marker, %d, and new state, %d)\n",
    yy_top_state(), xml_file_stack[xml_file_stack_len - 1].lexer_stack_marker, lexer_state));
  return xml_file_stack_len;

}


/*
 * push_xml_string
 *
 *   Pushes another scan object (in this case a string being inserted
 *   into the input stream, normally as a result of entity reference
 *   expansion) onto the xml_file_stack.  Unlike push_xml_file() and
 *   push_entity_text(), push_xml_string() pushes no new states onto
 *   the lexer state stack.  Returns the xml_file_stack_len.
 *
 *   This routine is used to re-scan expanded entity references.  It
 *   replaces pushback(), which could have been used instead, but was
 *   not (for performance reasons, mainly).  It is called solely from
 *   the lexer.
 */
static int
push_xml_string (struct xml_file *xf, my_wchar_t *wp)
{

  size_t size;

  if (++xml_file_stack_len > xml_file_stack_buflen)
    {
      size = sizeof (struct xml_file_state);
      if (xml_file_stack == NULL)
	{
	  xml_file_stack_buflen = 1;
	  if ((xml_file_stack = malloc (size * xml_file_stack_buflen)) == NULL)
	    errabort (40, "malloc() error in %s\n", "push_xml_file()");
	}
      else
	{
	  xml_file_stack_buflen = xml_file_stack_len;
	  if ((xml_file_stack = realloc (xml_file_stack, size * xml_file_stack_buflen)) == NULL)
	    errabort (41, "realloc() error in %s\n", "push_xml_file()");
	}
    }

  if (xml_file_stack_len == 1)
    {
      xml_file_stack[xml_file_stack_len - 1].buffer = YY_CURRENT_BUFFER;
      xml_file_stack[xml_file_stack_len - 1].what_object = file;
      xml_file_stack[xml_file_stack_len - 1].xf = this_file;
      xml_file_stack[xml_file_stack_len - 1].lexer_stack_marker = 0;
      return push_xml_string (xf, wp);
    }

  /* have flex read input from the child xml_file */
  xml_file_stack[xml_file_stack_len - 1].buffer = yy_scan_string (wp);
  /* no need to call xml_yy_switch_to_buffer () */
  xml_file_stack[xml_file_stack_len - 1].what_object = string;
  xml_file_stack[xml_file_stack_len - 1].xf = xf;
  xml_file_stack[xml_file_stack_len - 1].lexer_stack_marker = 0;
  this_file = xf;

  xwrap (errdebug (7, "Pushed XML string; now scanning it: \"%s\"\n", utf_16_to_utf_8 (wp)));
  xwrap (errdebug (7, "Lexer object stack len = %d\n", xml_file_stack_len));
  xwrap (errdebug (7, "Lexer state still = %d\n", YY_START));
  return xml_file_stack_len;

}


/*
 * pop_xml_file
 *
 *   Pop an object (not necessarily a file - so this is a misnomer) off
 *   the xml_file_stack (again a misnomer).  Returns the new length of
 *   the xml_file_stack.
 *
 *   Note that if the object being popped is not a string, this routine
 *   also pops off lexer states until it reaches the marker pushed onto
 *   the lexer state stack when the object was first pushed on.
 */
int
pop_xml_file (void)
{

  int stack_marker;
  enum scan_objects what_were_we_scanning;

  if (xml_file_stack_len == 0)
    {
      xwrap (errdebug (7, "Already in top-level file.  No object to pop.\n"));
      xwrap (errdebug (7, "Lexer object stack len = %d\n", xml_file_stack_len));
      xwrap (errdebug (7, "Lexer state still %d\n", YY_START));
    }
  else
    {
      /* file, entity text, or string ? */
      what_were_we_scanning = what_are_we_scanning ();

      /* pop states until we get back to our marker */
      stack_marker = xml_file_stack[xml_file_stack_len - 1].lexer_stack_marker;

      --xml_file_stack_len;

      if (xml_file_stack_len == 0)
	{
	  xwrap (errdebug (7, "Restored top-level object, \"%s\"\n", this_file->filename));
	  xwrap (errdebug (7, "Lexer object stack len = %d\n", xml_file_stack_len));
	  xwrap (errdebug (7, "Lexer state still %d\n", YY_START));
	}
      else
	{
	  /* delete the current flex buffer... */
          xml_yy_delete_buffer (xml_file_stack[xml_file_stack_len].buffer);

	  /* ...and switch to the previous one */
	  xml_yy_switch_to_buffer (xml_file_stack[xml_file_stack_len - 1].buffer);
	  this_file = xml_file_stack[xml_file_stack_len - 1].xf;

	  switch (what_were_we_scanning)
	    {
	    case entity_replacement_text:
	    case file:
	      /* push_xml_file() pushes a stack marker onto the lexer state stack */
	      while (YY_START != stack_marker)
		{
		  xwrap (errdebug (7, "popping off state %d\n", YY_START));
		  yy_pop_state ();
		}
	      xwrap (errdebug (7, "popping off state %d\n", YY_START));
	      yy_pop_state ();
	      xwrap (errdebug (7, "Popped XML object; now scanning \"%s\"\n", this_file->filename));
	      xwrap (errdebug (7, "Lexer object stack len = %d\n", xml_file_stack_len));
	      xwrap (errdebug (7, "Lexer state now back to %d\n", YY_START));
	      break;
	    case string:
	      /* push_xml_string() doesn't push a stack marker onto the lexer state stack */
	      xwrap (errdebug (7, "Popped XML object; now scanning previous object again.\n"));
	      xwrap (errdebug (7, "Lexer object stack len = %d\n", xml_file_stack_len));
	      xwrap (errdebug (7, "Lexer state still %d\n", YY_START));
	      break;
	    }
	}
    }

  return xml_file_stack_len;

}


/*
 * pushback
 *
 *   Simple utility for pushing back a string for re-scanning by the lexer.
 *   Slow; may overrun the current buffer and cause flex to abort.  Normally
 *   push_xml_string() is used instead.
 */
static int
pushback (my_wchar_t *wstr)
{

  int i, len;

  for (len = 0; wstr[len] != 0; len++);

  if (len == 0)
    return 0;

  for (i = len - 1; i >= 0; i--)
    unput (wstr[i]);

  /* What did we just do? */
  xwrap (errdebug (7, "pushed \"%s\" back onto lexer stream.\n", utf_16_to_utf_8 (wstr)));

  return len - 1;

}


/*
 * check_parend_nesting
 *
 *   Checks parameter-entity replacement text to see if it is nested
 *   correctly with respect to parentheses.  Also checks to be sure
 *   that neither the first nor last nonblank characters are a comma
 *   or vertical slash.  See the XML 1.0 spec, s. 3.2.1.
 *
 *   Returns 1 if everything is nested right, and ',' or '|' do not
 *   begin or end the unpadded part of the entity.  Also returns 1
 *   if '<' precedes '(', ')', ',' or '|', since in this case, we are
 *   looking at a parameter-entity used for markup.
 */
int
check_parend_nesting (my_wchar_t *entname, my_wchar_t *wp)
{

  size_t i;
  int errors, pcount;

  if (*wp != ' ')
    xwrap (errdebug (3, "Hmmm; PE replacment text isn't padded.\n"));

  while (uni_isspace (*wp))
    wp++;

  if (*wp == 0)
    {
      /* for interoperability, can't consist of all blanks (or nothing
       * at all)
       */
      add_xml_warning (this_file, 763, entname);
      return 0;
    }

  errors = 0;
  switch (*wp)
    {
    case ',':
    case '|':
      /* oops; first non-blank character is a comma or slash */
      add_xml_warning (this_file, 760, uni_truncate_to (wp, 2));
      errors = 1;
      break;
    case '<':
      /* callers will pick this up as a syntax error */
      return 1;
    }

  pcount = 0;
  for (i = 0; wp[i] != 0; i++)
    {
      switch (wp[i])
	{
	case '(':
	  pcount++;
	  break;
	case ')':
	  pcount--;
	  break;
	default:
	  break;
	}
    }

  if (pcount)
    {
      /* unbalanced parentheses */
      add_xml_error (this_file, 762, utf_8_to_utf_16 ((pcount > 0) ? "(" : ")"));
      errors = 1;
    }

  if (i)
    while ((wp[i] == 0 || uni_isspace (wp[i])) && i > 0)
      i--;
  if (wp[i] == ',' || wp[i] == '|')
    {
      /* last non-blank character is a comma or slash */
      add_xml_warning (this_file, 761, uni_truncate_to (&wp[i], 2));
      errors = 1;
    }

  /* If there were no errors, return 1; otherwise, 0. */
  return ! errors;

}


/*
 * xml_yywrap
 *
 *   Normally this routine is used to switch input streams in the
 *   lexer when EOF is reached (or terminate processing if there are
 *   no more files to read).  Here, though, the pattern <<EOF>> and
 *   the parser itself (w/ help from <<EOF>>) handle stream-switching.
 *   This is just a dummy routine that always tells the lexer to stop
 *   processing.
 */
int
xml_yywrap (void)
{
  return 1;
}