From fb099394d982c25f1e781526272b8e049a9a4510 Mon Sep 17 00:00:00 2001 From: Josh Price Date: Thu, 18 Jul 2024 13:40:49 +1000 Subject: [PATCH 1/2] Support tabs as whitespace --- lib/makeup/lexers/elixir_lexer.ex | 2 +- .../lexers/elixir_lexer/elixir_lexer_tokenizer_test.exs | 7 ++++++- 2 files changed, 7 insertions(+), 2 deletions(-) diff --git a/lib/makeup/lexers/elixir_lexer.ex b/lib/makeup/lexers/elixir_lexer.ex index 4d3404e..267e339 100644 --- a/lib/makeup/lexers/elixir_lexer.ex +++ b/lib/makeup/lexers/elixir_lexer.ex @@ -25,7 +25,7 @@ defmodule Makeup.Lexers.ElixirLexer do # TODO: check we're following this convention # NOTE: if Elixir had a good static type system it would help us do the right thing here. - whitespace = ascii_string([?\r, ?\s, ?\n, ?\f], min: 1) |> token(:whitespace) + whitespace = ascii_string([?\r, ?\s, ?\n, ?\f, ?\t], min: 1) |> token(:whitespace) newlines = optional(ascii_string([?\s, ?\t, ?\r], min: 1)) diff --git a/test/makeup/lexers/elixir_lexer/elixir_lexer_tokenizer_test.exs b/test/makeup/lexers/elixir_lexer/elixir_lexer_tokenizer_test.exs index 3ac5c4d..7fccdcc 100644 --- a/test/makeup/lexers/elixir_lexer/elixir_lexer_tokenizer_test.exs +++ b/test/makeup/lexers/elixir_lexer/elixir_lexer_tokenizer_test.exs @@ -2,8 +2,13 @@ defmodule ElixirLexerTokenizerTestSnippet do use ExUnit.Case, async: false import Makeup.Lexers.ElixirLexer.Testing, only: [lex: 1] - test "?\\f is recognized as whitespace character" do + test "whitespace" do + assert lex(" ") == [{:whitespace, %{}, " "}] + assert lex("\s") == [{:whitespace, %{}, "\s"}] + assert lex("\n") == [{:whitespace, %{}, "\n"}] + assert lex("\t") == [{:whitespace, %{}, "\t"}] assert lex("\f") == [{:whitespace, %{}, "\f"}] + assert lex("\r") == [{:whitespace, %{}, "\r"}] end test "builtins" do From e1e9d678112daa008cbd1ab8b9bc2ad6f470a393 Mon Sep 17 00:00:00 2001 From: Josh Price Date: Thu, 18 Jul 2024 21:24:22 +1000 Subject: [PATCH 2/2] Remove as not supported --- lib/makeup/lexers/elixir_lexer.ex | 2 +- test/makeup/lexers/elixir_lexer/elixir_lexer_tokenizer_test.exs | 1 - 2 files changed, 1 insertion(+), 2 deletions(-) diff --git a/lib/makeup/lexers/elixir_lexer.ex b/lib/makeup/lexers/elixir_lexer.ex index 267e339..48e17b6 100644 --- a/lib/makeup/lexers/elixir_lexer.ex +++ b/lib/makeup/lexers/elixir_lexer.ex @@ -25,7 +25,7 @@ defmodule Makeup.Lexers.ElixirLexer do # TODO: check we're following this convention # NOTE: if Elixir had a good static type system it would help us do the right thing here. - whitespace = ascii_string([?\r, ?\s, ?\n, ?\f, ?\t], min: 1) |> token(:whitespace) + whitespace = ascii_string([?\r, ?\s, ?\n, ?\t], min: 1) |> token(:whitespace) newlines = optional(ascii_string([?\s, ?\t, ?\r], min: 1)) diff --git a/test/makeup/lexers/elixir_lexer/elixir_lexer_tokenizer_test.exs b/test/makeup/lexers/elixir_lexer/elixir_lexer_tokenizer_test.exs index 7fccdcc..f93d185 100644 --- a/test/makeup/lexers/elixir_lexer/elixir_lexer_tokenizer_test.exs +++ b/test/makeup/lexers/elixir_lexer/elixir_lexer_tokenizer_test.exs @@ -7,7 +7,6 @@ defmodule ElixirLexerTokenizerTestSnippet do assert lex("\s") == [{:whitespace, %{}, "\s"}] assert lex("\n") == [{:whitespace, %{}, "\n"}] assert lex("\t") == [{:whitespace, %{}, "\t"}] - assert lex("\f") == [{:whitespace, %{}, "\f"}] assert lex("\r") == [{:whitespace, %{}, "\r"}] end