Translate wide strings

cpp2rust/converter/converter.cpp

@@ -2210,6 +2210,13 @@ bool Converter::VisitFloatingLiteral(clang::FloatingLiteral *expr) {
 }
 
 bool Converter::VisitCharacterLiteral(clang::CharacterLiteral *expr) {
+  if (expr->getKind() != clang::CharacterLiteralKind::Ascii) {
+    PushParen paren(*this);
+    StrCat(std::to_string(expr->getValue()), keyword::kAs,
+           ToStringBase(expr->getType()));
+    computed_expr_type_ = ComputedExprType::FreshValue;
+    return false;
+  }
   auto uc = static_cast<unsigned char>(expr->getValue());
   std::string ch = GetEscapedCharLiteral(expr->getValue());
   ch = (uc > 0x7F ? "b'" : "'") + std::move(ch) + '\'';
@@ -2274,7 +2281,37 @@ std::string Converter::GetEscapedStringLiteral(clang::Expr *expr,
   return out;
 }
 
+bool Converter::IsArrayInitContext() const {
+  return !curr_init_type_.empty() && curr_init_type_.back()->isArrayType();
+}
+
+std::string
+Converter::GetCodeUnitArrayLiteral(const clang::StringLiteral *expr) {
+  auto elem_type =
+      ToStringBase(ctx_.getAsArrayType(expr->getType())->getElementType());
+  uint64_t len = expr->getLength();
+  uint64_t total = len + 1;
+  if (IsArrayInitContext()) {
+    if (auto *arr_ty = ctx_.getAsConstantArrayType(curr_init_type_.back())) {
+      total = std::max(arr_ty->getSize().getZExtValue(), len);
+    }
+  }
+  std::string out = '[';
+  for (uint64_t i = 0; i < total; ++i) {
+    out += std::format("{} as {}, ", i < len ? expr->getCodeUnit(i) : 0,
+                       elem_type);
+  }
+  out += ']';
+  return out;
+}
+
 bool Converter::VisitStringLiteral(clang::StringLiteral *expr) {
+  if (IsCodeUnitStringLiteral(expr)) {
+    StrCat(GetCodeUnitArrayLiteral(expr));
+    computed_expr_type_ = ComputedExprType::FreshValue;
+    return false;
+  }
+
   auto init_type = curr_init_type_.empty()
                        ? clang::QualType()
                        : curr_init_type_.back().getNonReferenceType();

cpp2rust/converter/converter.h

@@ -349,6 +349,8 @@ class Converter : public clang::RecursiveASTVisitor<Converter> {
   virtual bool VisitCharacterLiteral(clang::CharacterLiteral *expr);
 
   std::string GetEscapedCharLiteral(char character) const;
+  std::string GetCodeUnitArrayLiteral(const clang::StringLiteral *expr);
+  bool IsArrayInitContext() const;
 
   std::string GetEscapedUTF8CharLiteral(clang::Expr *expr) const;
 

cpp2rust/converter/converter_lib.cpp

@@ -149,6 +149,11 @@ bool IsStringLiteralExpr(const clang::Expr *expr) {
          clang::isa<clang::PredefinedExpr>(stripped);
 }
 
+bool IsCodeUnitStringLiteral(const clang::StringLiteral *expr) {
+  return expr->getCharByteWidth() != 1 ||
+         expr->getKind() == clang::StringLiteralKind::UTF8;
+}
+
 bool IsUserDefinedDecl(const clang::Decl *decl) {
   const auto &ctx = decl->getASTContext();
   const auto &src_mgr = ctx.getSourceManager();

cpp2rust/converter/converter_lib.h

@@ -46,6 +46,8 @@ bool IsUnionArrayMember(const clang::Expr *base);
 
 bool IsStringLiteralExpr(const clang::Expr *expr);
 
+bool IsCodeUnitStringLiteral(const clang::StringLiteral *expr);
+
 bool IsUserDefinedDecl(const clang::Decl *decl);
 
 bool RefersToUserDefinedDecl(const clang::Expr *expr);

cpp2rust/converter/models/converter_refcount.cpp

@@ -1179,7 +1179,15 @@ bool ConverterRefCount::VisitCallExpr(clang::CallExpr *expr) {
 }
 
 bool ConverterRefCount::VisitStringLiteral(clang::StringLiteral *expr) {
-  if (!curr_init_type_.empty() && curr_init_type_.back()->isArrayType()) {
+  if (IsCodeUnitStringLiteral(expr)) {
+    auto arr = GetCodeUnitArrayLiteral(expr);
+    StrCat(IsArrayInitContext() ? std::format("Box::from({})", arr)
+                                : '&' + arr);
+    computed_expr_type_ = ComputedExprType::FreshValue;
+    return false;
+  }
+
+  if (IsArrayInitContext()) {
     uint64_t pad = 1;
     if (auto *arr_ty = ctx_.getAsConstantArrayType(curr_init_type_.back())) {
       uint64_t arr_size = arr_ty->getSize().getZExtValue();
@@ -1272,7 +1280,11 @@ bool ConverterRefCount::VisitImplicitCastExpr(clang::ImplicitCastExpr *expr) {
       return false;
     }
     if (IsStringLiteralExpr(sub_expr)) {
-      StrCat(std::format("Ptr::from_string_literal({})",
+      auto code_unit = ToStringBase(
+          ctx_.getAsArrayType(
+                  sub_expr->IgnoreParens()->IgnoreImplicit()->getType())
+              ->getElementType());
+      StrCat(std::format("Ptr::<{}>::from_string_literal({})", code_unit,
                          ToString(sub_expr->IgnoreParens())));
       computed_expr_type_ = ComputedExprType::FreshPointer;
       return false;
@@ -2144,7 +2156,12 @@ std::string ConverterRefCount::ConvertVarInitValue(clang::QualType qual_type,
     }
     if (qual_type.getNonReferenceType()->isArrayType()) {
       if (IsStringLiteralExpr(expr)) {
-        return std::format("Ptr::from_string_literal_array({})",
+        auto code_unit = ToStringBase(
+            ctx_.getAsArrayType(
+                    expr->IgnoreParens()->IgnoreImplicit()->getType())
+                ->getElementType());
+        return std::format("Ptr::<Box<[{}]>>::from_string_literal_array({})",
+                           code_unit,
                            ToString(expr->IgnoreParens()->IgnoreImplicit()));
       }
       return std::format("({} as {})", ConvertFreshPointer(expr),
@@ -2414,7 +2431,12 @@ void ConverterRefCount::ConvertArraySubscript(clang::Expr *base,
     {
       PushParen paren(*this, is_inner_boxed);
       if (IsStringLiteralExpr(base)) {
-        StrCat(std::format("Ptr::from_string_literal({}).offset({})",
+        auto code_unit = ToStringBase(
+            ctx_.getAsArrayType(
+                    base->IgnoreParens()->IgnoreImplicit()->getType())
+                ->getElementType());
+        StrCat(std::format("Ptr::<{}>::from_string_literal({}).offset({})",
+                           code_unit,
                            ToString(base->IgnoreParens()->IgnoreImplicit()),
                            ConvertSubscriptIndex(idx)));
       } else {

libcc2rs/src/cstr.rs

@@ -27,32 +27,49 @@ impl fmt::Display for Ptr<u8> {
     }
 }
 
-type StringLiteralMap = HashMap<&'static [u8], Rc<RefCell<Box<[u8]>>>>;
+macro_rules! impl_string_literal {
+    ($t:ty, $cache:ident) => {
+        thread_local! {
+            static $cache: RefCell<HashMap<&'static [$t], Rc<RefCell<Box<[$t]>>>>> =
+                RefCell::new(HashMap::new());
+        }
 
-thread_local! {
-    static STRING_LITERALS: RefCell<StringLiteralMap> = RefCell::new(HashMap::new());
-}
+        impl Ptr<Box<[$t]>> {
+            #[inline]
+            pub fn from_string_literal_array(s: &'static [$t]) -> Self {
+                $cache.with(|literals| {
+                    let mut literals = literals.borrow_mut();
+                    let weak = Rc::downgrade(literals.entry(s).or_insert_with(|| {
+                        Rc::new(RefCell::new({
+                            let mut v = s.to_vec();
+                            v.push(0);
+                            v.into_boxed_slice()
+                        }))
+                    }));
+                    Ptr {
+                        offset: 0,
+                        kind: PtrKind::StackSingle(weak),
+                    }
+                })
+            }
+        }
 
-impl Ptr<Box<[u8]>> {
-    #[inline]
-    pub fn from_string_literal_array(s: &'static [u8]) -> Self {
-        STRING_LITERALS.with(|literals| {
-            let mut literals = literals.borrow_mut();
-            let weak = Rc::downgrade(literals.entry(s).or_insert_with(|| {
-                Rc::new(RefCell::new({
-                    let mut v = s.to_vec();
-                    v.push(0);
-                    v.into_boxed_slice()
-                }))
-            }));
-            Ptr {
-                offset: 0,
-                kind: PtrKind::StackSingle(weak),
+        impl Ptr<$t> {
+            #[inline]
+            pub fn from_string_literal(s: &'static [$t]) -> Self {
+                Ptr::<Box<[$t]>>::from_string_literal_array(s)
+                    .to_strong()
+                    .as_pointer()
             }
-        })
-    }
+        }
+    };
 }
 
+impl_string_literal!(u8, STRING_LITERALS_U8);
+impl_string_literal!(u16, STRING_LITERALS_U16);
+impl_string_literal!(u32, STRING_LITERALS_U32);
+impl_string_literal!(i32, STRING_LITERALS_I32);
+
 impl Ptr<u8> {
     #[allow(clippy::explicit_counter_loop)]
     pub fn memcpy(&self, src: &Self, len: usize) {
@@ -104,13 +121,6 @@ impl Ptr<u8> {
         0
     }
 
-    #[inline]
-    pub fn from_string_literal(s: &'static [u8]) -> Self {
-        Ptr::<Box<[u8]>>::from_string_literal_array(s)
-            .to_strong()
-            .as_pointer()
-    }
-
     pub fn to_c_string_iterator(&self) -> CStringIterator {
         CStringIterator { ptr: self.clone() }
     }

libcc2rs/src/libc_shims/time.rs

@@ -37,7 +37,7 @@ impl Tm {
         let zone: &'static [u8] = b"GMT";
         #[cfg(target_os = "macos")]
         let zone: &'static [u8] = b"UTC";
-        *tm.tm_zone.borrow_mut() = Ptr::from_string_literal(zone);
+        *tm.tm_zone.borrow_mut() = Ptr::<u8>::from_string_literal(zone);
         tm
     }
 

rules/locale/tgt_refcount.rs

@@ -8,5 +8,5 @@ use libcc2rs::*;
 // TODO: we need to track ourselves the locale settings and change the behavior of the relevant
 // functions based on the set locale.
 fn f1(a0: i32, a1: Ptr<u8>) -> Ptr<u8> {
-    Ptr::from_string_literal(b"C")
+    Ptr::<u8>::from_string_literal(b"C")
 }

tests/ub/out/refcount/fd_double_close.rs

@@ -17,7 +17,7 @@ fn main_0() -> i32 {
             None => nix::sys::stat::Mode::empty(),
         };
         match nix::fcntl::open(
-            Ptr::from_string_literal(b"/dev/null")
+            Ptr::<u8>::from_string_literal(b"/dev/null")
                 .to_rust_string()
                 .as_str(),
             nix::fcntl::OFlag::from_bits_retain(::libc::O_RDONLY),

tests/ub/out/refcount/fd_use_after_close.rs

@@ -17,7 +17,7 @@ fn main_0() -> i32 {
             None => nix::sys::stat::Mode::empty(),
         };
         match nix::fcntl::open(
-            Ptr::from_string_literal(b"/dev/null")
+            Ptr::<u8>::from_string_literal(b"/dev/null")
                 .to_rust_string()
                 .as_str(),
             nix::fcntl::OFlag::from_bits_retain(::libc::O_RDONLY),

tests/ub/out/refcount/file_use_after_fclose.rs

@@ -13,8 +13,8 @@ pub fn main() {
 fn main_0() -> i32 {
     let fp: Value<Ptr<CFile>> = Rc::new(RefCell::new(
         match CFile::open(
-            &Ptr::from_string_literal(b"/tmp/cpp2rust_uafc_test.tmp").to_rust_string(),
-            &Ptr::from_string_literal(b"wb").to_rust_string(),
+            &Ptr::<u8>::from_string_literal(b"/tmp/cpp2rust_uafc_test.tmp").to_rust_string(),
+            &Ptr::<u8>::from_string_literal(b"wb").to_rust_string(),
         ) {
             Some(__f) => Ptr::alloc(__f),
             None => Ptr::null(),

tests/unit/out/refcount/array_reference.rs

@@ -164,7 +164,7 @@ pub fn main() {
     std::process::exit(main_0());
 }
 fn main_0() -> i32 {
-    assert!((({ len_0(Ptr::from_string_literal_array(b"beta"),) }) == 4));
+    assert!((({ len_0(Ptr::<Box<[u8]>>::from_string_literal_array(b"beta"),) }) == 4));
     let buf: Value<Box<[u8]>> = Rc::new(RefCell::new(Box::from(*b"abcd\0")));
     assert!((({ len_0((buf.as_pointer() as Ptr<Box<[u8]>>),) }) == 4));
     let arr: Value<Box<[i32]>> = Rc::new(RefCell::new(Box::new([1, 2, 3])));
@@ -176,10 +176,10 @@ fn main_0() -> i32 {
     ({ fill_and_sum_5((arr.as_pointer() as Ptr<Box<[i32]>>), 2, out.as_pointer()) });
     assert!(((*out.borrow()) == 12));
     assert!(((*arr.borrow())[(0) as usize] == 2));
-    let lit: Ptr<Box<[u8]>> = Ptr::from_string_literal_array(b"beta");
+    let lit: Ptr<Box<[u8]>> = Ptr::<Box<[u8]>>::from_string_literal_array(b"beta");
     assert!((({ len_0(((lit).clone() as Ptr<Box<[u8]>>),) }) == 4));
     assert!(
-        ((((({ pick_6(Ptr::from_string_literal_array(b"beta"),) })
+        ((((({ pick_6(Ptr::<Box<[u8]>>::from_string_literal_array(b"beta"),) })
             .to_strong()
             .as_pointer() as Ptr::<u8>)
             .offset((0) as isize)
@@ -206,8 +206,8 @@ fn main_0() -> i32 {
     assert!(((*(*pts.borrow())[(1) as usize].y.borrow()) == 14));
     assert!((({ sum_points_7((pts.as_pointer() as Ptr<Box<[Point]>>),) }) == 30));
     let names: Value<Box<[Ptr<u8>]>> = Rc::new(RefCell::new(Box::new([
-        Ptr::from_string_literal(b"ab"),
-        Ptr::from_string_literal(b"cde"),
+        Ptr::<u8>::from_string_literal(b"ab"),
+        Ptr::<u8>::from_string_literal(b"cde"),
     ])));
     assert!((({ total_len_9((names.as_pointer() as Ptr<Box<[Ptr::<u8>]>>),) }) == 5));
     return 0;

tests/unit/out/refcount/bool_condition_logical.rs

@@ -104,7 +104,7 @@ fn main_0() -> i32 {
     if ((*n.borrow()) != 0) || (((*bits.borrow()) & 256_i64) != 0) {
         assert!(true);
     }
-    let cp: Value<Ptr<u8>> = Rc::new(RefCell::new(Ptr::from_string_literal(b"hi")));
+    let cp: Value<Ptr<u8>> = Rc::new(RefCell::new(Ptr::<u8>::from_string_literal(b"hi")));
     let cnp: Value<Ptr<u8>> = Rc::new(RefCell::new(Ptr::<u8>::null()));
     if ((*x.borrow()) > (*y.borrow())) && (!(*cp.borrow()).is_null()) {
         assert!(true);

tests/unit/out/refcount/bool_condition_logical_c.rs

@@ -127,7 +127,7 @@ fn main_0() -> i32 {
     {
         assert!((1 != 0));
     }
-    let cp: Value<Ptr<u8>> = Rc::new(RefCell::new(Ptr::from_string_literal(b"hi")));
+    let cp: Value<Ptr<u8>> = Rc::new(RefCell::new(Ptr::<u8>::from_string_literal(b"hi")));
     let cnp: Value<Ptr<u8>> = Rc::new(RefCell::new(Ptr::<u8>::null()));
     if (((((((*x.borrow()) > (*y.borrow())) as i32) != 0) && (!(*cp.borrow()).is_null())) as i32)
         != 0)

tests/unit/out/refcount/char_printing.rs

@@ -14,7 +14,7 @@ fn main_0() -> i32 {
     let vec_: Value<Vec<u8>> = Rc::new(RefCell::new(vec![195_u8, 167_u8]));
     let i: Value<i32> = Rc::new(RefCell::new(27));
     let str: Value<Vec<u8>> = Rc::new(RefCell::new(
-        Ptr::from_string_literal(b"rdas.")
+        Ptr::<u8>::from_string_literal(b"rdas.")
             .to_c_string_iterator()
             .chain(std::iter::once(0))
             .collect::<Vec<u8>>(),

tests/unit/out/refcount/char_printing_cerr.rs

@@ -14,7 +14,7 @@ fn main_0() -> i32 {
     let vec_: Value<Vec<u8>> = Rc::new(RefCell::new(vec![195_u8, 167_u8]));
     let i: Value<i32> = Rc::new(RefCell::new(27));
     let str: Value<Vec<u8>> = Rc::new(RefCell::new(
-        Ptr::from_string_literal(b"rdas.")
+        Ptr::<u8>::from_string_literal(b"rdas.")
             .to_c_string_iterator()
             .chain(std::iter::once(0))
             .collect::<Vec<u8>>(),