Modernize plpgsql's handling of parse locations, making it look a lot more

like the core parser's code.  In particular, track locations at the character
rather than line level during parsing, allowing many more parse-time error
conditions to be reported with precise error pointers rather than just
"near line N".

Also, exploit the fact that we no longer need to substitute $N for variable
references by making extracted SQL queries and expressions be exact copies
of subranges of the function text, rather than having random whitespace
changes within them.  This makes it possible to directly map parse error
positions from the core parser onto positions in the function text, which
lets us report them without the previous kluge of showing the intermediate
internal-query form.  (Later it might be good to do that for core
parse-analysis errors too, but this patch is just touching plpgsql's
lexer/parser, not what happens at runtime.)

In passing, make plpgsql's lexer use palloc not malloc.

These changes make plpgsql's parse-time error reports noticeably nicer
(as illustrated by the regression test changes), and will also simplify
the planned removal of plpgsql's separate lexer by reducing the impedance
mismatch between what it does and what the core lexer does.

src/pl/plpgsql/src/pl_comp.c

@@ -8,7 +8,7 @@
  *
  *
  * IDENTIFICATION
- *	  $PostgreSQL: pgsql/src/pl/plpgsql/src/pl_comp.c,v 1.142 2009/11/07 00:52:26 tgl Exp $
+ *	  $PostgreSQL: pgsql/src/pl/plpgsql/src/pl_comp.c,v 1.143 2009/11/09 00:26:55 tgl Exp $
  *
  *-------------------------------------------------------------------------
  */
@@ -46,7 +46,6 @@ int			plpgsql_nDatums;
 PLpgSQL_datum **plpgsql_Datums;
 static int	datums_last = 0;
 
-int			plpgsql_error_lineno;
 char	   *plpgsql_error_funcname;
 bool		plpgsql_DumpExecTree = false;
 bool		plpgsql_check_syntax = false;
@@ -95,6 +94,7 @@ static PLpgSQL_function *do_compile(FunctionCallInfo fcinfo,
 		   PLpgSQL_function *function,
 		   PLpgSQL_func_hashkey *hashkey,
 		   bool forValidator);
+static void plpgsql_compile_error_callback(void *arg);
 static void add_dummy_return(PLpgSQL_function *function);
 static Node *plpgsql_pre_column_ref(ParseState *pstate, ColumnRef *cref);
 static Node *plpgsql_post_column_ref(ParseState *pstate, ColumnRef *cref, Node *var);
@@ -301,7 +301,6 @@ do_compile(FunctionCallInfo fcinfo,
 	plpgsql_scanner_init(proc_source);
 
 	plpgsql_error_funcname = pstrdup(NameStr(procStruct->proname));
-	plpgsql_error_lineno = 0;
 
 	/*
 	 * Setup error traceback support for ereport()
@@ -713,7 +712,6 @@ do_compile(FunctionCallInfo fcinfo,
 	 */
 	error_context_stack = plerrcontext.previous;
 	plpgsql_error_funcname = NULL;
-	plpgsql_error_lineno = 0;
 
 	plpgsql_check_syntax = false;
 
@@ -752,7 +750,6 @@ plpgsql_compile_inline(char *proc_source)
 	plpgsql_scanner_init(proc_source);
 
 	plpgsql_error_funcname = func_name;
-	plpgsql_error_lineno = 0;
 
 	/*
 	 * Setup error traceback support for ereport()
@@ -851,7 +848,6 @@ plpgsql_compile_inline(char *proc_source)
 	 */
 	error_context_stack = plerrcontext.previous;
 	plpgsql_error_funcname = NULL;
-	plpgsql_error_lineno = 0;
 
 	plpgsql_check_syntax = false;
 
@@ -865,10 +861,8 @@ plpgsql_compile_inline(char *proc_source)
  * error context callback to let us supply a call-stack traceback.
  * If we are validating or executing an anonymous code block, the function
  * source text is passed as an argument.
- *
- * This function is public only for the sake of an assertion in gram.y
  */
-void
+static void
 plpgsql_compile_error_callback(void *arg)
 {
 	if (arg)
@@ -888,7 +882,7 @@ plpgsql_compile_error_callback(void *arg)
 
 	if (plpgsql_error_funcname)
 		errcontext("compilation of PL/pgSQL function \"%s\" near line %d",
-				   plpgsql_error_funcname, plpgsql_error_lineno);
+				   plpgsql_error_funcname, plpgsql_latest_lineno());
 }
 
 
@@ -2065,25 +2059,6 @@ build_row_from_vars(PLpgSQL_variable **vars, int numvars)
 	return row;
 }
 
-
-/* ----------
- * plpgsql_parse_datatype			Scanner found something that should
- *					be a datatype name.
- * ----------
- */
-PLpgSQL_type *
-plpgsql_parse_datatype(const char *string)
-{
-	Oid			type_id;
-	int32		typmod;
-
-	/* Let the main parser try to parse it under standard SQL rules */
-	parseTypeString(string, &type_id, &typmod);
-
-	/* Okay, build a PLpgSQL_type data structure for it */
-	return plpgsql_build_datatype(type_id, typmod);
-}
-
 /*
  * plpgsql_build_datatype
  *		Build PLpgSQL_type struct given type OID and typmod.

src/pl/plpgsql/src/pl_exec.c

@@ -8,7 +8,7 @@
  *
  *
  * IDENTIFICATION
- *	  $PostgreSQL: pgsql/src/pl/plpgsql/src/pl_exec.c,v 1.250 2009/11/06 18:37:54 tgl Exp $
+ *	  $PostgreSQL: pgsql/src/pl/plpgsql/src/pl_exec.c,v 1.251 2009/11/09 00:26:55 tgl Exp $
  *
  *-------------------------------------------------------------------------
  */
@@ -23,7 +23,6 @@
 #include "catalog/pg_type.h"
 #include "executor/spi_priv.h"
 #include "funcapi.h"
-#include "lib/stringinfo.h"
 #include "miscadmin.h"
 #include "nodes/nodeFuncs.h"
 #include "parser/scansup.h"

src/pl/plpgsql/src/plpgsql.h

@@ -8,7 +8,7 @@
  *
  *
  * IDENTIFICATION
- *	  $PostgreSQL: pgsql/src/pl/plpgsql/src/plpgsql.h,v 1.121 2009/11/07 00:52:26 tgl Exp $
+ *	  $PostgreSQL: pgsql/src/pl/plpgsql/src/plpgsql.h,v 1.122 2009/11/09 00:26:55 tgl Exp $
  *
  *-------------------------------------------------------------------------
  */
@@ -22,6 +22,7 @@
 #include "fmgr.h"
 #include "commands/trigger.h"
 #include "executor/spi.h"
+#include "lib/stringinfo.h"
 #include "nodes/bitmapset.h"
 #include "utils/tuplestore.h"
 
@@ -774,11 +775,9 @@ typedef struct
 
 extern bool plpgsql_DumpExecTree;
 extern bool plpgsql_LookupIdentifiers;
-extern bool plpgsql_SpaceScanned;
 extern int	plpgsql_nDatums;
 extern PLpgSQL_datum **plpgsql_Datums;
 
-extern int	plpgsql_error_lineno;
 extern char *plpgsql_error_funcname;
 
 /* linkage to the real yytext variable */
@@ -813,7 +812,6 @@ extern PLpgSQL_type *plpgsql_parse_dblwordtype(const char *word);
 extern PLpgSQL_type *plpgsql_parse_tripwordtype(const char *word);
 extern PLpgSQL_type *plpgsql_parse_wordrowtype(const char *word);
 extern PLpgSQL_type *plpgsql_parse_dblwordrowtype(const char *word);
-extern PLpgSQL_type *plpgsql_parse_datatype(const char *string);
 extern PLpgSQL_type *plpgsql_build_datatype(Oid typeOid, int32 typmod);
 extern PLpgSQL_variable *plpgsql_build_variable(const char *refname, int lineno,
 					   PLpgSQL_type *dtype,
@@ -826,7 +824,6 @@ extern PLpgSQL_condition *plpgsql_parse_err_condition(char *condname);
 extern void plpgsql_adddatum(PLpgSQL_datum *new);
 extern int	plpgsql_add_initdatums(int **varnos);
 extern void plpgsql_HashTableInit(void);
-extern void plpgsql_compile_error_callback(void *arg);
 
 /* ----------
  * Functions in pl_handler.c
@@ -885,8 +882,12 @@ extern int	plpgsql_yyparse(void);
 extern int	plpgsql_base_yylex(void);
 extern int	plpgsql_yylex(void);
 extern void plpgsql_push_back_token(int token);
+extern void plpgsql_append_source_text(StringInfo buf,
+									   int startlocation, int endlocation);
+extern int	plpgsql_scanner_errposition(int location);
 extern void plpgsql_yyerror(const char *message);
-extern int	plpgsql_scanner_lineno(void);
+extern int	plpgsql_location_to_lineno(int location);
+extern int	plpgsql_latest_lineno(void);
 extern void plpgsql_scanner_init(const char *str);
 extern void plpgsql_scanner_finish(void);
 

src/pl/plpgsql/src/scan.l

 %{
 /*-------------------------------------------------------------------------
  *
- * scan.l		- Scanner for the PL/pgSQL
- *			  procedural language
+ * scan.l		- Scanner for the PL/pgSQL procedural language
  *
  * Portions Copyright (c) 1996-2009, PostgreSQL Global Development Group
  * Portions Copyright (c) 1994, Regents of the University of California
  *
  *
  * IDENTIFICATION
- *	  $PostgreSQL: pgsql/src/pl/plpgsql/src/scan.l,v 1.74 2009/11/07 00:52:26 tgl Exp $
+ *	  $PostgreSQL: pgsql/src/pl/plpgsql/src/scan.l,v 1.75 2009/11/09 00:26:55 tgl Exp $
  *
  *-------------------------------------------------------------------------
  */
@@ -24,27 +23,32 @@
 #define fprintf(file, fmt, msg)  ereport(ERROR, (errmsg_internal("%s", msg)))
 
 /*
+ * Each call to yylex must set yylloc to the location of the found token
+ * (expressed as a byte offset from the start of the input text).
  * When we parse a token that requires multiple lexer rules to process,
- * remember the token's starting position this way.
+ * this should be done in the first such rule, else yylloc will point
+ * into the middle of the token.
  */
-#define SAVE_TOKEN_START()  \
-	( start_lineno = plpgsql_scanner_lineno(), start_charpos = yytext )
+#define SET_YYLLOC()  (yylloc = yytext - scanbuf)
 
 /* Handles to the buffer that the lexer uses internally */
 static YY_BUFFER_STATE scanbufhandle;
 static char *scanbuf;
 
-static const char *scanstr;		/* original input string */
+static const char *scanorig;		/* original input string */
 
 static int	pushback_token;
 static bool have_pushback_token;
 static const char *cur_line_start;
+static const char *cur_line_end;
 static int	cur_line_num;
 static int		xcdepth = 0;	/* depth of nesting in slash-star comments */
 static char    *dolqstart;      /* current $foo$ quote start string */
 
 bool plpgsql_LookupIdentifiers = true;
-bool plpgsql_SpaceScanned = false;
+
+static void location_lineno_init(void);
+
 %}
 
 %option 8bit
@@ -53,6 +57,10 @@ bool plpgsql_SpaceScanned = false;
 %option noinput
 %option nounput
 %option noyywrap
+%option noyyalloc
+%option noyyrealloc
+%option noyyfree
+%option warn
 %option prefix="plpgsql_base_yy"
 
 %option case-insensitive
@@ -126,133 +134,117 @@ param			\${digit}+
 
 %%
     /* ----------
-     * Local variables in scanner to remember where
-     * a string or comment started
-     * ----------
-     */
-    int	start_lineno = 0;
-	char *start_charpos = NULL;
-
-    /* ----------
-     * Reset the state when entering the scanner
+     * Reset the state when entering yylex()
      * ----------
      */
     BEGIN(INITIAL);
-    plpgsql_SpaceScanned = false;
 
     /* ----------
      * The keyword rules
      * ----------
      */
-:=				{ return K_ASSIGN;			}
-=				{ return K_ASSIGN;			}
-\.\.			{ return K_DOTDOT;			}
-alias			{ return K_ALIAS;			}
-all				{ return K_ALL;				}
-begin			{ return K_BEGIN;			}
-by				{ return K_BY;   			}
-case			{ return K_CASE;			}
-close			{ return K_CLOSE;			}
-constant		{ return K_CONSTANT;		}
-continue		{ return K_CONTINUE;		}
-cursor			{ return K_CURSOR;			}
-declare			{ return K_DECLARE;			}
-default			{ return K_DEFAULT;			}
-diagnostics		{ return K_DIAGNOSTICS;		}
-else			{ return K_ELSE;			}
-elseif			{ return K_ELSIF;			}
-elsif			{ return K_ELSIF;			}
-end				{ return K_END;				}
-exception		{ return K_EXCEPTION;		}
-execute			{ return K_EXECUTE;			}
-exit			{ return K_EXIT;			}
-fetch			{ return K_FETCH;			}
-for				{ return K_FOR;				}
-from			{ return K_FROM;			}
-get				{ return K_GET;				}
-if				{ return K_IF;				}
-in				{ return K_IN;				}
-insert			{ return K_INSERT;			}
-into			{ return K_INTO;			}
-is				{ return K_IS;				}
-loop			{ return K_LOOP;			}
-move			{ return K_MOVE;			}
-no{space}+scroll { return K_NOSCROLL;		}
-not				{ return K_NOT;				}
-null			{ return K_NULL;			}
-open			{ return K_OPEN;			}
-or				{ return K_OR;				}
-perform			{ return K_PERFORM;			}
-raise			{ return K_RAISE;			}
-result_oid		{ return K_RESULT_OID;		}
-return			{ return K_RETURN;			}
-reverse			{ return K_REVERSE;			}
-row_count		{ return K_ROW_COUNT;		}
-scroll			{ return K_SCROLL;			}
-strict			{ return K_STRICT;	    	}
-then			{ return K_THEN;			}
-to				{ return K_TO;				}
-type			{ return K_TYPE;			}
-using			{ return K_USING;			}
-when			{ return K_WHEN;			}
-while			{ return K_WHILE;			}
-
-^#option		{ return O_OPTION;			}
-dump			{ return O_DUMP;			}
+:=				{ SET_YYLLOC(); return K_ASSIGN;		}
+=				{ SET_YYLLOC(); return K_ASSIGN;		}
+\.\.			{ SET_YYLLOC(); return K_DOTDOT;		}
+alias			{ SET_YYLLOC(); return K_ALIAS;			}
+all				{ SET_YYLLOC(); return K_ALL;			}
+begin			{ SET_YYLLOC(); return K_BEGIN;			}
+by				{ SET_YYLLOC(); return K_BY;   			}
+case			{ SET_YYLLOC(); return K_CASE;			}
+close			{ SET_YYLLOC(); return K_CLOSE;			}
+constant		{ SET_YYLLOC(); return K_CONSTANT;		}
+continue		{ SET_YYLLOC(); return K_CONTINUE;		}
+cursor			{ SET_YYLLOC(); return K_CURSOR;		}
+declare			{ SET_YYLLOC(); return K_DECLARE;		}
+default			{ SET_YYLLOC(); return K_DEFAULT;		}
+diagnostics		{ SET_YYLLOC(); return K_DIAGNOSTICS;	}
+else			{ SET_YYLLOC(); return K_ELSE;			}
+elseif			{ SET_YYLLOC(); return K_ELSIF;			}
+elsif			{ SET_YYLLOC(); return K_ELSIF;			}
+end				{ SET_YYLLOC(); return K_END;			}
+exception		{ SET_YYLLOC(); return K_EXCEPTION;		}
+execute			{ SET_YYLLOC(); return K_EXECUTE;		}
+exit			{ SET_YYLLOC(); return K_EXIT;			}
+fetch			{ SET_YYLLOC(); return K_FETCH;			}
+for				{ SET_YYLLOC(); return K_FOR;			}
+from			{ SET_YYLLOC(); return K_FROM;			}
+get				{ SET_YYLLOC(); return K_GET;			}
+if				{ SET_YYLLOC(); return K_IF;			}
+in				{ SET_YYLLOC(); return K_IN;			}
+insert			{ SET_YYLLOC(); return K_INSERT;		}
+into			{ SET_YYLLOC(); return K_INTO;			}
+is				{ SET_YYLLOC(); return K_IS;			}
+loop			{ SET_YYLLOC(); return K_LOOP;			}
+move			{ SET_YYLLOC(); return K_MOVE;			}
+no{space}+scroll { SET_YYLLOC(); return K_NOSCROLL;		}
+not				{ SET_YYLLOC(); return K_NOT;			}
+null			{ SET_YYLLOC(); return K_NULL;			}
+open			{ SET_YYLLOC(); return K_OPEN;			}
+or				{ SET_YYLLOC(); return K_OR;			}
+perform			{ SET_YYLLOC(); return K_PERFORM;		}
+raise			{ SET_YYLLOC(); return K_RAISE;			}
+result_oid		{ SET_YYLLOC(); return K_RESULT_OID;	}
+return			{ SET_YYLLOC(); return K_RETURN;		}
+reverse			{ SET_YYLLOC(); return K_REVERSE;		}
+row_count		{ SET_YYLLOC(); return K_ROW_COUNT;		}
+scroll			{ SET_YYLLOC(); return K_SCROLL;		}
+strict			{ SET_YYLLOC(); return K_STRICT;	    }
+then			{ SET_YYLLOC(); return K_THEN;			}
+to				{ SET_YYLLOC(); return K_TO;			}
+type			{ SET_YYLLOC(); return K_TYPE;			}
+using			{ SET_YYLLOC(); return K_USING;			}
+when			{ SET_YYLLOC(); return K_WHEN;			}
+while			{ SET_YYLLOC(); return K_WHILE;			}
+
+^#option		{ SET_YYLLOC(); return O_OPTION;		}
+dump			{ SET_YYLLOC(); return O_DUMP;			}
 
 
     /* ----------
      * Special word rules
-	 *
-	 * We set plpgsql_error_lineno in each rule so that errors reported
-	 * in the pl_comp.c subroutines will point to the right place.
      * ----------
      */
 {identifier}					{
-	plpgsql_error_lineno = plpgsql_scanner_lineno();
+	SET_YYLLOC();
 	if (!plpgsql_LookupIdentifiers) return T_WORD;
 	return plpgsql_parse_word(yytext); }
 {identifier}{space}*\.{space}*{identifier}	{
-	plpgsql_error_lineno = plpgsql_scanner_lineno();
+	SET_YYLLOC();
 	if (!plpgsql_LookupIdentifiers) return T_DBLWORD;
 	return plpgsql_parse_dblword(yytext); }
 {identifier}{space}*\.{space}*{identifier}{space}*\.{space}*{identifier}	{
-	plpgsql_error_lineno = plpgsql_scanner_lineno();
+	SET_YYLLOC();
 	if (!plpgsql_LookupIdentifiers) return T_TRIPWORD;
 	return plpgsql_parse_tripword(yytext); }
 {param}							{
-	plpgsql_error_lineno = plpgsql_scanner_lineno();
+	SET_YYLLOC();
 	if (!plpgsql_LookupIdentifiers) return T_WORD;
 	return plpgsql_parse_word(yytext); }
 {param}{space}*\.{space}*{identifier}	{
-	plpgsql_error_lineno = plpgsql_scanner_lineno();
+	SET_YYLLOC();
 	if (!plpgsql_LookupIdentifiers) return T_DBLWORD;
 	return plpgsql_parse_dblword(yytext); }
 {param}{space}*\.{space}*{identifier}{space}*\.{space}*{identifier}	{
-	plpgsql_error_lineno = plpgsql_scanner_lineno();
+	SET_YYLLOC();
 	if (!plpgsql_LookupIdentifiers) return T_TRIPWORD;
 	return plpgsql_parse_tripword(yytext); }
 
-{digit}+		{ return T_NUMBER;			}
+{digit}+		{ SET_YYLLOC(); return T_NUMBER;		}
 
-\".				{ yyerror("unterminated quoted identifier"); }
-
-    /* ----------
-     * Ignore whitespace (including comments) but remember this happened
-     * ----------
-     */
-{whitespace}	{ plpgsql_SpaceScanned = true; }
+\".				{ SET_YYLLOC(); yyerror("unterminated quoted identifier"); }
 
     /* ----------
      * Comment and literal handling is mostly copied from the core lexer
      * ----------
      */
+{whitespace}	{
+					/* ignore */
+				}
+
 {xcstart}		{
-					/* Set location in case of syntax error in comment */
-					SAVE_TOKEN_START();
+					SET_YYLLOC();
 					xcdepth = 0;
 					BEGIN(xc);
-					plpgsql_SpaceScanned = true;
 				}
 
 <xc>{xcstart}	{
@@ -281,14 +273,14 @@ dump			{ return O_DUMP;			}
 <xc><<EOF>>		{ yyerror("unterminated /* comment"); }
 
 {xqstart}		{
-					SAVE_TOKEN_START();
+					SET_YYLLOC();
 					if (standard_conforming_strings)
 						BEGIN(xq);
 					else
 						BEGIN(xe);
 				}
 {xestart}		{
-					SAVE_TOKEN_START();
+					SET_YYLLOC();
 					BEGIN(xe);
 				}
 <xq,xe>{quotestop}	|
@@ -296,8 +288,8 @@ dump			{ return O_DUMP;			}
 					yyless(1);
 					BEGIN(INITIAL);
 					/* adjust yytext/yyleng to describe whole string token */
-					yyleng += (yytext - start_charpos);
-					yytext = start_charpos;
+					yyleng += (yytext - (scanbuf + yylloc));
+					yytext = scanbuf + yylloc;
 					return T_STRING;
 				}
 <xq,xe>{xqdouble} {
@@ -317,7 +309,7 @@ dump			{ return O_DUMP;			}
 <xq,xe><<EOF>>		{ yyerror("unterminated quoted string"); }
 
 {dolqdelim}		{
-					SAVE_TOKEN_START();
+					SET_YYLLOC();
 					dolqstart = pstrdup(yytext);
 					BEGIN(xdolq);
 				}
@@ -325,7 +317,7 @@ dump			{ return O_DUMP;			}
 					/* throw back all but the initial "$" */
 					yyless(1);
 					/* and treat it as {other} */
-					return yytext[0];
+					SET_YYLLOC(); return yytext[0];
 				}
 <xdolq>{dolqdelim} {
 					if (strcmp(yytext, dolqstart) == 0)
@@ -333,8 +325,8 @@ dump			{ return O_DUMP;			}
 						pfree(dolqstart);
 						BEGIN(INITIAL);
 						/* adjust yytext/yyleng to describe whole string */
-						yyleng += (yytext - start_charpos);
-						yytext = start_charpos;
+						yyleng += (yytext - (scanbuf + yylloc));
+						yytext = scanbuf + yylloc;
 						return T_STRING;
 					}
 					else
@@ -361,7 +353,7 @@ dump			{ return O_DUMP;			}
      * ----------
      */
 .				{
-					return yytext[0];
+					SET_YYLLOC(); return yytext[0];
 				}
 
 %%
@@ -369,9 +361,10 @@ dump			{ return O_DUMP;			}
 
 /*
  * This is the yylex routine called from outside. It exists to provide
- * a one-token pushback facility.  Beware of trying to make it do more:
- * for the most part, plpgsql's gram.y assumes that yytext is in step
- * with the "current token".
+ * a one-token pushback facility.  Beware of trying to push back more;
+ * for the most part, plpgsql's gram.y assumes that yytext and yylloc
+ * are in step with the "current token".  In particular it is assumed that
+ * those are in step with the result immediately after any yylex() call.
  */
 int
 plpgsql_yylex(void)
@@ -387,7 +380,8 @@ plpgsql_yylex(void)
 /*
  * Push back a single token to be re-read by next plpgsql_yylex() call.
  *
- * NOTE: this does not cause yytext to "back up".
+ * NOTE: this does not cause yytext or yylloc to "back up".  Also, it
+ * is not a good idea to push back a token other than what you read.
  */
 void
 plpgsql_push_back_token(int token)
@@ -399,18 +393,61 @@ plpgsql_push_back_token(int token)
 }
 
 /*
- * Report a syntax error.
+ * Append the function text starting at startlocation and extending to
+ * (not including) endlocation onto the existing contents of "buf".
  */
 void
-plpgsql_yyerror(const char *message)
+plpgsql_append_source_text(StringInfo buf,
+						   int startlocation, int endlocation)
 {
-	const char *loc = yytext;
-	int			cursorpos;
+	Assert(startlocation <= endlocation);
+	appendBinaryStringInfo(buf, scanorig + startlocation,
+						   endlocation - startlocation);
+}
 
-	plpgsql_error_lineno = plpgsql_scanner_lineno();
+/*
+ * plpgsql_scanner_errposition
+ *		Report an error cursor position, if possible.
+ *
+ * This is expected to be used within an ereport() call.  The return value
+ * is a dummy (always 0, in fact).
+ *
+ * Note that this can only be used for messages emitted during initial
+ * parsing of a plpgsql function, since it requires the scanorig string
+ * to still be available.
+ */
+int
+plpgsql_scanner_errposition(int location)
+{
+	int		pos;
 
-	/* in multibyte encodings, return index in characters not bytes */
-	cursorpos = pg_mbstrlen_with_len(scanbuf, loc - scanbuf) + 1;
+	if (location < 0 || scanorig == NULL)
+		return 0;				/* no-op if location is unknown */
+
+	/* Convert byte offset to character number */
+	pos = pg_mbstrlen_with_len(scanorig, location) + 1;
+	/* And pass it to the ereport mechanism */
+	(void) internalerrposition(pos);
+	/* Also pass the function body string */
+	return internalerrquery(scanorig);
+}
+
+/*
+ * plpgsql_yyerror
+ *		Report a lexer or grammar error.
+ *
+ * The message's cursor position is whatever YYLLOC was last set to,
+ * ie, the start of the current token if called within yylex(), or the
+ * most recently lexed token if called from the grammar.
+ * This is OK for syntax error messages from the Bison parser, because Bison
+ * parsers report error as soon as the first unparsable token is reached.
+ * Beware of using yyerror for other purposes, as the cursor position might
+ * be misleading!
+ */
+void
+plpgsql_yyerror(const char *message)
+{
+	const char *loc = scanbuf + yylloc;
 
 	if (*loc == YY_END_OF_BUFFER_CHAR)
 	{
@@ -418,8 +455,7 @@ plpgsql_yyerror(const char *message)
 				(errcode(ERRCODE_SYNTAX_ERROR),
 				 /* translator: %s is typically the translation of "syntax error" */
 				 errmsg("%s at end of input", _(message)),
-				 internalerrposition(cursorpos),
-				 internalerrquery(scanstr)));
+				 plpgsql_scanner_errposition(yylloc)));
 	}
 	else
 	{
@@ -427,33 +463,72 @@ plpgsql_yyerror(const char *message)
 				(errcode(ERRCODE_SYNTAX_ERROR),
 				 /* translator: first %s is typically the translation of "syntax error" */
 				 errmsg("%s at or near \"%s\"", _(message), loc),
-				 internalerrposition(cursorpos),
-				 internalerrquery(scanstr)));
+				 plpgsql_scanner_errposition(yylloc)));
 	}
 }
 
 /*
- * Get the line number at which the current token ends.  This substitutes
- * for flex's very poorly implemented yylineno facility.
+ * Given a location (a byte offset in the function source text),
+ * return a line number.
  *
- * We assume that flex has written a '\0' over the character following the
- * current token in scanbuf.  So, we just have to count the '\n' characters
- * before that.  We optimize this a little by keeping track of the last
- * '\n' seen so far.
+ * We expect that this is typically called for a sequence of increasing
+ * location values, so optimize accordingly by tracking the endpoints
+ * of the "current" line.
  */
 int
-plpgsql_scanner_lineno(void)
+plpgsql_location_to_lineno(int location)
 {
-	const char *c;
+	const char *loc;
+
+	if (location < 0 || scanorig == NULL)
+		return 0;				/* garbage in, garbage out */
+	loc = scanorig + location;
 
-	while ((c = strchr(cur_line_start, '\n')) != NULL)
+	/* be correct, but not fast, if input location goes backwards */
+	if (loc < cur_line_start)
+		location_lineno_init();
+
+	while (cur_line_end != NULL && loc > cur_line_end)
 	{
-		cur_line_start = c + 1;
+		cur_line_start = cur_line_end + 1;
 		cur_line_num++;
+		cur_line_end = strchr(cur_line_start, '\n');
 	}
+
+	return cur_line_num;
+}
+
+/* initialize or reset the state for plpgsql_location_to_lineno */
+static void
+location_lineno_init(void)
+{
+	cur_line_start = scanorig;
+	cur_line_num = 1;
+
+	/*----------
+	 * Hack: skip any initial newline, so that in the common coding layout
+	 *		CREATE FUNCTION ... AS $$
+	 *			code body
+	 *		$$ LANGUAGE plpgsql;
+	 * we will think "line 1" is what the programmer thinks of as line 1.
+	 *----------
+	 */
+    if (*cur_line_start == '\r')
+        cur_line_start++;
+    if (*cur_line_start == '\n')
+        cur_line_start++;
+
+	cur_line_end = strchr(cur_line_start, '\n');
+}
+
+/* return the most recently computed lineno */
+int
+plpgsql_latest_lineno(void)
+{
 	return cur_line_num;
 }
 
+
 /*
  * Called before any actual parsing is done
  *
@@ -464,48 +539,37 @@ plpgsql_scanner_lineno(void)
 void
 plpgsql_scanner_init(const char *str)
 {
-	Size	slen;
-
-	slen = strlen(str);
+	Size	slen = strlen(str);
 
 	/*
-	 * Might be left over after ereport()
+	 * Reset flex internal state.  Whatever data it might think it has
+	 * has long since been pfree'd.
 	 */
-	if (YY_CURRENT_BUFFER)
-		yy_delete_buffer(YY_CURRENT_BUFFER);
+	yy_init_globals();
 
 	/*
 	 * Make a scan buffer with special termination needed by flex.
 	 */
-	scanbuf = palloc(slen + 2);
+	scanbuf = (char *) palloc(slen + 2);
 	memcpy(scanbuf, str, slen);
 	scanbuf[slen] = scanbuf[slen + 1] = YY_END_OF_BUFFER_CHAR;
 	scanbufhandle = yy_scan_buffer(scanbuf, slen + 2);
 
-	/* Other setup */
-	scanstr = str;
+	/*
+	 * scanorig points to the original string, which unlike scanbuf won't
+	 * be modified on-the-fly by flex.  Notice that although yytext points
+	 * into scanbuf, we rely on being able to apply locations (offsets from
+	 * string start) to scanorig as well.
+	 */
+	scanorig = str;
 
+	/* Other setup */
 	have_pushback_token = false;
 
-	cur_line_start = scanbuf;
-	cur_line_num = 1;
-
-	/*----------
-	 * Hack: skip any initial newline, so that in the common coding layout
-	 *		CREATE FUNCTION ... AS '
-	 *			code body
-	 *		' LANGUAGE plpgsql;
-	 * we will think "line 1" is what the programmer thinks of as line 1.
-	 *----------
-	 */
-    if (*cur_line_start == '\r')
-        cur_line_start++;
-    if (*cur_line_start == '\n')
-        cur_line_start++;
+	location_lineno_init();
 
 	BEGIN(INITIAL);
 	plpgsql_LookupIdentifiers = true;
-	plpgsql_SpaceScanned = false;
 }
 
 /*
@@ -514,6 +578,38 @@ plpgsql_scanner_init(const char *str)
 void
 plpgsql_scanner_finish(void)
 {
+	/* release storage */
 	yy_delete_buffer(scanbufhandle);
 	pfree(scanbuf);
+	/* avoid leaving any dangling pointers */
+	scanbufhandle = NULL;
+	scanbuf = NULL;
+	scanorig = NULL;
+}
+
+/*
+ * Interface functions to make flex use palloc() instead of malloc().
+ * It'd be better to make these static, but flex insists otherwise.
+ */
+
+void *
+plpgsql_base_yyalloc(yy_size_t bytes)
+{
+	return palloc(bytes);
+}
+
+void *
+plpgsql_base_yyrealloc(void *ptr, yy_size_t bytes)
+{
+	if (ptr)
+		return repalloc(ptr, bytes);
+	else
+		return palloc(bytes);
+}
+
+void
+plpgsql_base_yyfree(void *ptr)
+{
+	if (ptr)
+		pfree(ptr);
 }

src/test/regress/expected/plpgsql.out

@@ -1747,7 +1747,7 @@ create function f1(in i int, out j int) returns int as $$
 begin
   return i+1;
 end$$ language plpgsql;
-ERROR:  RETURN cannot have a parameter in function with OUT parameters at or near "i"
+ERROR:  RETURN cannot have a parameter in function with OUT parameters
 LINE 3:   return i+1;
                  ^
 create function f1(in i int, out j int) as $$
@@ -2066,13 +2066,13 @@ begin
 end$$ language plpgsql;
 select test_variable_storage();
 NOTICE:  should see this
-CONTEXT:  SQL statement "SELECT  trap_zero_divide(-100)"
+CONTEXT:  SQL statement "SELECT trap_zero_divide(-100)"
 PL/pgSQL function "test_variable_storage" line 7 at PERFORM
 NOTICE:  should see this only if -100 <> 0
-CONTEXT:  SQL statement "SELECT  trap_zero_divide(-100)"
+CONTEXT:  SQL statement "SELECT trap_zero_divide(-100)"
 PL/pgSQL function "test_variable_storage" line 7 at PERFORM
 NOTICE:  should see this only if -100 fits in smallint
-CONTEXT:  SQL statement "SELECT  trap_zero_divide(-100)"
+CONTEXT:  SQL statement "SELECT trap_zero_divide(-100)"
 PL/pgSQL function "test_variable_storage" line 7 at PERFORM
  test_variable_storage 
 -----------------------
@@ -2325,10 +2325,8 @@ begin
     return a;
 end$$ language plpgsql;
 ERROR:  syntax error at or near "Johnny"
-LINE 1: Johnny Yuma
-        ^
-QUERY:  Johnny Yuma
-CONTEXT:  SQL statement in PL/PgSQL function "bad_sql1" near line 4
+LINE 5:     Johnny Yuma;
+            ^
 create function bad_sql2() returns int as $$
 declare r record;
 begin
@@ -2338,26 +2336,22 @@ begin
     return 5;
 end;$$ language plpgsql;
 ERROR:  syntax error at or near "the"
-LINE 1:  select I fought the law, the law won
-                         ^
-QUERY:   select I fought the law, the law won
-CONTEXT:  SQL statement in PL/PgSQL function "bad_sql2" near line 3
+LINE 4:     for r in select I fought the law, the law won LOOP
+                                     ^
 -- a RETURN expression is mandatory, except for void-returning
 -- functions, where it is not allowed
 create function missing_return_expr() returns int as $$
 begin
     return ;
 end;$$ language plpgsql;
-ERROR:  syntax error at end of input
-LINE 1: SELECT 
-               ^
-QUERY:  SELECT 
-CONTEXT:  SQL statement in PL/PgSQL function "missing_return_expr" near line 2
+ERROR:  missing expression at or near ";"
+LINE 3:     return ;
+                   ^
 create function void_return_expr() returns void as $$
 begin
     return 5;
 end;$$ language plpgsql;
-ERROR:  RETURN cannot have a parameter in function returning void at or near "5"
+ERROR:  RETURN cannot have a parameter in function returning void
 LINE 3:     return 5;
                    ^
 -- VOID functions are allowed to omit RETURN
@@ -2427,9 +2421,9 @@ end; $$ language plpgsql;
 -- blocks
 select excpt_test1();
 ERROR:  column "sqlstate" does not exist
-LINE 1: SELECT  sqlstate
-                ^
-QUERY:  SELECT  sqlstate
+LINE 1: SELECT sqlstate
+               ^
+QUERY:  SELECT sqlstate
 CONTEXT:  PL/pgSQL function "excpt_test1" line 2 at RAISE
 create function excpt_test2() returns void as $$
 begin
@@ -2442,9 +2436,9 @@ end; $$ language plpgsql;
 -- should fail
 select excpt_test2();
 ERROR:  column "sqlstate" does not exist
-LINE 1: SELECT  sqlstate
-                ^
-QUERY:  SELECT  sqlstate
+LINE 1: SELECT sqlstate
+               ^
+QUERY:  SELECT sqlstate
 CONTEXT:  PL/pgSQL function "excpt_test2" line 4 at RAISE
 create function excpt_test3() returns void as $$
 begin
@@ -2714,7 +2708,8 @@ begin
 end;
 $$ language plpgsql;
 ERROR:  end label "outer_label" differs from block's label "inner_label"
-CONTEXT:  compilation of PL/pgSQL function "end_label3" near line 6
+LINE 7:   end loop outer_label;
+                   ^
 -- should fail: end label on a block without a start label
 create function end_label4() returns void as $$
 <<outer_label>>
@@ -2725,7 +2720,8 @@ begin
 end;
 $$ language plpgsql;
 ERROR:  end label "outer_label" specified for unlabelled block
-CONTEXT:  compilation of PL/pgSQL function "end_label4" near line 5
+LINE 6:   end loop outer_label;
+                   ^
 -- using list of scalars in fori and fore stmts
 create function for_vect() returns void as $proc$
 <<lbl>>declare a integer; b varchar; c varchar; r record;
@@ -3308,7 +3304,8 @@ begin
 end;
 $$ language plpgsql;
 ERROR:  cursor FOR loop must use a bound cursor variable
-CONTEXT:  compilation of PL/pgSQL function "forc_bad" near line 4
+LINE 5:   for r in c loop
+                   ^
 -- test RETURN QUERY EXECUTE
 create or replace function return_dquery()
 returns setof int as $$
@@ -3839,21 +3836,20 @@ begin
 end
 $$ language plpgsql;
 WARNING:  nonstandard use of \\ in a string literal
+LINE 3:   raise notice 'foo\\bar\041baz';
+                       ^
 HINT:  Use the escape string syntax for backslashes, e.g., E'\\'.
-CONTEXT:  string literal in PL/PgSQL function "strtest" near line 2
 WARNING:  nonstandard use of \\ in a string literal
-LINE 1: SELECT  'foo\\bar\041baz'
-                ^
+LINE 4:   return 'foo\\bar\041baz';
+                 ^
 HINT:  Use the escape string syntax for backslashes, e.g., E'\\'.
-QUERY:  SELECT  'foo\\bar\041baz'
-CONTEXT:  SQL statement in PL/PgSQL function "strtest" near line 3
 select strtest();
 NOTICE:  foo\bar!baz
 WARNING:  nonstandard use of \\ in a string literal
-LINE 1: SELECT  'foo\\bar\041baz'
-                ^
+LINE 1: SELECT 'foo\\bar\041baz'
+               ^
 HINT:  Use the escape string syntax for backslashes, e.g., E'\\'.
-QUERY:  SELECT  'foo\\bar\041baz'
+QUERY:  SELECT 'foo\\bar\041baz'
 CONTEXT:  PL/pgSQL function "strtest" line 3 at RETURN
    strtest   
 -------------
@@ -3922,7 +3918,7 @@ NOTICE:  105, Office
 NOTICE:  106, Office
 -- these are to check syntax error reporting
 DO LANGUAGE plpgsql $$begin return 1; end$$;
-ERROR:  RETURN cannot have a parameter in function returning void at or near "1"
+ERROR:  RETURN cannot have a parameter in function returning void
 LINE 1: DO LANGUAGE plpgsql $$begin return 1; end$$;
                                            ^
 DO LANGUAGE plpgsql $$
@@ -3934,9 +3930,9 @@ BEGIN
     END LOOP;
 END$$;
 ERROR:  column "foo" does not exist
-LINE 1:  SELECT rtrim(roomno) AS roomno, foo FROM Room ORDER BY room...
-                                         ^
-QUERY:   SELECT rtrim(roomno) AS roomno, foo FROM Room ORDER BY roomno
+LINE 1: SELECT rtrim(roomno) AS roomno, foo FROM Room ORDER BY roomn...
+                                        ^
+QUERY:  SELECT rtrim(roomno) AS roomno, foo FROM Room ORDER BY roomno
 CONTEXT:  PL/pgSQL function "inline_code_block" line 3 at FOR over SELECT rows
 -- Check variable scoping -- a var is not available in its own or prior
 -- default expressions.