From c21177b113dec0b936cf6f87a4f95ff2cb069502 Mon Sep 17 00:00:00 2001
From: John MacFarlane <jgm@berkeley.edu>
Date: Thu, 13 Feb 2025 13:57:25 -0800
Subject: [PATCH] Smart quote parsing: ignore curly quotes.

Previously we tried to match curly quotes as well as straight
quotes, producing Quoted inlines.  But it seems better just to
assume that those who use curly quotes want them passed through
verbatim.

This also fixes an (unintended) bug whereby curly single left
quotes would sometimes be changed to single right quotes.

Closes #10610.
---
 src/Text/Pandoc/Parsing/Smart.hs      | 16 ++++------------
 src/Text/Pandoc/Writers/LaTeX/Util.hs |  4 ++--
 test/command/5407.md                  |  8 ++++----
 test/command/pandoc-citeproc-320a.md  | 12 ++++++------
 test/markdown-reader-more.native      |  4 ++--
 5 files changed, 18 insertions(+), 26 deletions(-)

diff --git a/src/Text/Pandoc/Parsing/Smart.hs b/src/Text/Pandoc/Parsing/Smart.hs
index 879251905..29c19bb77 100644
--- a/src/Text/Pandoc/Parsing/Smart.hs
+++ b/src/Text/Pandoc/Parsing/Smart.hs
@@ -43,7 +43,6 @@ import Text.Parsec
   , notFollowedBy
   , try
   )
-import qualified Data.Text as T
 import qualified Text.Pandoc.Builder as B
 
 -- | Parses various ASCII punctuation, quotes, and apostrophe in a smart
@@ -93,13 +92,6 @@ doubleQuoted inlineParser = do
      (withQuoteContext InDoubleQuote (manyTill inlineParser doubleQuoteEnd)))
    <|> pure (B.str "\8220")
 
-charOrRef :: (Stream s m Char, UpdateSourcePos s Char) => [Char] -> ParsecT s st m Char
-charOrRef cs =
-  oneOf cs <|> try (do t <- characterReference
-                       case T.unpack t of
-                         [c] | c `elem` cs -> return c
-                         _ -> fail "unexpected character reference")
-
 -- | Succeeds if the parser is
 --
 -- * not within single quoted text;
@@ -116,13 +108,13 @@ singleQuoteStart = do
   -- single quote start can't be right after str
   guard =<< notAfterString
   try $ do
-    charOrRef "'\8216\145"
+    char '\''
     void $ lookAhead (satisfy (not . isSpaceChar))
 
 singleQuoteEnd :: (Stream s m Char, UpdateSourcePos s Char)
                => ParsecT s st m ()
 singleQuoteEnd = try $ do
-  charOrRef "'\8217\146"
+  char '\''
   notFollowedBy alphaNum
 
 -- | Succeeds if the parser is
@@ -142,13 +134,13 @@ doubleQuoteStart :: (HasLastStrPosition st,
 doubleQuoteStart = do
   failIfInQuoteContext InDoubleQuote
   guard =<< notAfterString
-  try $ do charOrRef "\"\8220\147"
+  try $ do char '"'
            void $ lookAhead (satisfy (not . isSpaceChar))
 
 -- | Parses a closing quote character.
 doubleQuoteEnd :: (Stream s m Char, UpdateSourcePos s Char)
                => ParsecT s st m ()
-doubleQuoteEnd = void (charOrRef "\"\8221\148")
+doubleQuoteEnd = void (char '"')
 
 -- | Parses an ASCII apostrophe (@'@) or right single quotation mark and
 -- returns a RIGHT SINGLE QUOtatiON MARK character.
diff --git a/src/Text/Pandoc/Writers/LaTeX/Util.hs b/src/Text/Pandoc/Writers/LaTeX/Util.hs
index 5cccb82f2..da5fb9cd3 100644
--- a/src/Text/Pandoc/Writers/LaTeX/Util.hs
+++ b/src/Text/Pandoc/Writers/LaTeX/Util.hs
@@ -108,11 +108,11 @@ stringToLaTeX context zs = do
          '}' -> emits "\\}"
          '?' | ligatures ->  -- avoid ?` ligature
            case xs of
-             '`':_ -> emits "?{}"
+             '`':_ -> emits "?{\\kern0pt}" -- se #10610
              _     -> emitc x
          '!' | ligatures ->  -- avoid !` ligature
            case xs of
-             '`':_ -> emits "!{}"
+             '`':_ -> emits "!{\\kern0pt}"
              _     -> emitc x
          '`' | ctx == CodeString -> emitcseq "\\textasciigrave"
          '$' -> emits "\\$"
diff --git a/test/command/5407.md b/test/command/5407.md
index ac9bed29d..b8a37a804 100644
--- a/test/command/5407.md
+++ b/test/command/5407.md
@@ -6,9 +6,9 @@ hi there?‘
 hi there!‘
 hi there!
 ^D
-hi there?{}``
-hi there!{}``
-hi there?{}`
-hi there!{}`
+hi there?{\kern0pt}``
+hi there!{\kern0pt}``
+hi there?{\kern0pt}`
+hi there!{\kern0pt}`
 hi there!
 ```
diff --git a/test/command/pandoc-citeproc-320a.md b/test/command/pandoc-citeproc-320a.md
index 081dbfe3f..0afc769a5 100644
--- a/test/command/pandoc-citeproc-320a.md
+++ b/test/command/pandoc-citeproc-320a.md
@@ -49,13 +49,17 @@ references:
 Foo [@item1; @item2; @item3; @item4; @item5; @item6; @item7; @item8].
 ^D
 Foo (al-ʾUdhrī, n.d.; al-ʿUdhrī, n.d.; al-\'Udhrī, n.d.; al-'Udhrī,
-n.d.a, n.d.b; Uch, n.d.; Uebel, n.d.; Zzz, n.d.).
+n.d.; al-'Udhrī, n.d.; Uch, n.d.; Uebel, n.d.; Zzz, n.d.).
 
 ::::::::::: {#refs .references .csl-bib-body .hanging-indent entry-spacing="0"}
 ::: {#ref-item6 .csl-entry}
 Uch, Ann. n.d.
 :::
 
+::: {#ref-item4 .csl-entry}
+'Udhrī, Jamīl al-. n.d.
+:::
+
 ::: {#ref-item1 .csl-entry}
 ʾUdhrī, Jamīl al-. n.d.
 :::
@@ -68,12 +72,8 @@ Uch, Ann. n.d.
 \'Udhrī, Jamīl al-. n.d.
 :::
 
-::: {#ref-item4 .csl-entry}
-'Udhrī, Jamīl al-. n.d.a.
-:::
-
 ::: {#ref-item5 .csl-entry}
----------. n.d.b.
+'Udhrī, Jamīl al-. n.d.
 :::
 
 ::: {#ref-item7 .csl-entry}
diff --git a/test/markdown-reader-more.native b/test/markdown-reader-more.native
index 41a2e3345..148cba944 100644
--- a/test/markdown-reader-more.native
+++ b/test/markdown-reader-more.native
@@ -316,8 +316,8 @@ Pandoc
       2
       ( "curly-smart-quotes" , [] , [] )
       [ Str "Curly" , Space , Str "smart" , Space , Str "quotes" ]
-  , Para [ Quoted DoubleQuote [ Str "Hi" ] ]
-  , Para [ Quoted SingleQuote [ Str "Hi" ] ]
+  , Para [ Str "\8220Hi\8221" ]
+  , Para [ Str "\8216Hi\8217" ]
   , Header
       2
       ( "consecutive-lists" , [] , [] )