Offsetof fixes (#11690)

* first fixes

* more tests and fixes

* code normalization

compiler/ccgexprs.nim

@@ -2168,6 +2168,22 @@ proc genMagicExpr(p: BProc, e: PNode, d: var TLoc, op: TMagic) =
     if not p.module.compileToCpp:
       p.module.includeHeader("<stdalign.h>")
     putIntoDest(p, d, e, "((NI)alignof($1))" % [getTypeDesc(p.module, t)])
+  of mOffsetOf:
+    var dotExpr: PNode
+    block findDotExpr:
+      if e[1].kind == nkDotExpr:
+        dotExpr = e[1]
+      elif e[1].kind == nkCheckedFieldExpr:
+        dotExpr = e[1][0]
+      else:
+        internalError(p.config, e.info, "unknown ast")
+    let t = dotExpr[0].typ.skipTypes({tyTypeDesc})
+    let member =
+      if t.kind == tyTuple:
+        "Field" & rope(dotExpr[1].sym.position)
+      else:
+        rope(dotExpr[1].sym.name.s)
+    putIntoDest(p,d,e, "((NI)offsetof($1, $2))" % [getTypeDesc(p.module, t), member])
   of mChr: genSomeCast(p, e, d)
   of mOrd: genOrd(p, e, d)
   of mLengthArray, mHigh, mLengthStr, mLengthSeq, mLengthOpenArray:

compiler/semexprs.nim

@@ -2119,14 +2119,7 @@ proc semSizeof(c: PContext, n: PNode): PNode =
     n.sons[1] = semExprWithType(c, n.sons[1], {efDetermineType})
     #restoreOldStyleType(n.sons[1])
   n.typ = getSysType(c.graph, n.info, tyInt)
-
+  result = foldSizeOf(c.config, n, n)
-  let size = getSize(c.config, n[1].typ)
-  if size >= 0:
-    result = newIntNode(nkIntLit, size)
-    result.info = n.info
-    result.typ = n.typ
-  else:
-    result = n
 
 proc semMagic(c: PContext, n: PNode, s: PSym, flags: TExprFlags): PNode =
   # this is a hotspot in the compiler!
@@ -2215,7 +2208,8 @@ proc semMagic(c: PContext, n: PNode, s: PSym, flags: TExprFlags): PNode =
       result = setMs(n, s)
     else:
       result = c.graph.emptyNode
-  of mSizeOf: result = semSizeof(c, setMs(n, s))
+  of mSizeOf: result =
+    semSizeof(c, setMs(n, s))
   else:
     result = semDirectOp(c, n, flags)
 

compiler/semfold.nim

@@ -648,13 +648,11 @@ proc getConstExpr(m: PSym, n: PNode; g: ModuleGraph): PNode =
         # This fixes bug #544.
         result = newIntNodeT(lengthOrd(g.config, n.sons[1].typ), n, g)
       of mSizeOf:
-        let size = getSize(g.config, n[1].typ)
+        result = foldSizeOf(g.config, n, nil)
-        if size >= 0:
+      of mAlignOf:
-          result = newIntNode(nkIntLit, size)
+        result = foldAlignOf(g.config, n, nil)
-          result.info = n.info
+      of mOffsetOf:
-          result.typ = getSysType(g, n.info, tyInt)
+        result = foldOffsetOf(g.config, n, nil)
-        else:
-          result = nil
       of mAstToStr:
         result = newStrNodeT(renderTree(n[1], {renderNoComments}), n, g)
       of mConStrStr:

compiler/semmagic.nim

@@ -375,52 +375,11 @@ proc magicsAfterOverloadResolution(c: PContext, n: PNode,
   of mTypeOf:
     result = semTypeOf(c, n)
   of mSizeOf:
-    # TODO there is no proper way to find out if a type cannot be queried for the size.
+    result = foldSizeOf(c.config, n, n)
-    let size = getSize(c.config, n[1].typ)
-    # We just assume here that the type might come from the c backend
-    if size == szUnknownSize:
-      # Forward to the c code generation to emit a `sizeof` in the C code.
-      result = n
-    elif size >= 0:
-      result = newIntNode(nkIntLit, size)
-      result.info = n.info
-      result.typ = n.typ
-    else:
-      localError(c.config, n.info, "cannot evaluate 'sizeof' because its type is not defined completely, type: " & n[1].typ.typeToString)
-      result = n
   of mAlignOf:
-    # this is 100% analog to mSizeOf, could be made more dry.
+    result = foldAlignOf(c.config, n, n)
-    let align = getAlign(c.config, n[1].typ)
-    if align == szUnknownSize:
-      result = n
-    elif align >= 0:
-      result = newIntNode(nkIntLit, align)
-      result.info = n.info
-      result.typ = n.typ
-    else:
-      localError(c.config, n.info, "cannot evaluate 'alignof' because its type is not defined completely, type: " & n[1].typ.typeToString)
-      result = n
   of mOffsetOf:
-    var dotExpr: PNode
+    result = foldOffsetOf(c.config, n, n)
-
-    block findDotExpr:
-      if n[1].kind == nkDotExpr:
-        dotExpr = n[1]
-      elif n[1].kind == nkCheckedFieldExpr:
-        dotExpr = n[1][0]
-      else:
-        illFormedAst(n, c.config)
-
-    assert dotExpr != nil
-
-    let value = dotExpr[0]
-    let member = dotExpr[1]
-
-    discard computeSize(c.config, value.typ)
-
-    result = newIntNode(nkIntLit, member.sym.offset)
-    result.info = n.info
-    result.typ = n.typ
   of mArrGet:
     result = semArrGet(c, n, flags)
   of mArrPut:

compiler/sizealignoffsetimpl.nim

@@ -11,6 +11,10 @@
 proc align(address, alignment: BiggestInt): BiggestInt =
   result = (address + (alignment - 1)) and not (alignment - 1)
 
+proc align(address, alignment: int): int =
+  result = (address + (alignment - 1)) and not (alignment - 1)
+
+
 const
   ## a size is concidered "unknown" when it is an imported type from C
   ## or C++.
@@ -18,6 +22,38 @@ const
   szIllegalRecursion* = -2
   szUncomputedSize* = -1
 
+type IllegalTypeRecursionError = object of Exception
+
+proc raiseIllegalTypeRecursion() =
+  raise newException(IllegalTypeRecursionError, "illegal type recursion")
+
+type
+  OffsetAccum = object
+    maxAlign: int
+    offset: int
+
+proc inc(arg: var OffsetAccum; value: int) =
+  if unlikely(value == szIllegalRecursion):  raiseIllegalTypeRecursion()
+  if value == szUnknownSize or arg.offset == szUnknownSize:
+    arg.offset = szUnknownSize
+  else:
+    arg.offset += value
+
+proc align(arg: var OffsetAccum; value: int) =
+  if unlikely(value == szIllegalRecursion):  raiseIllegalTypeRecursion()
+  if value == szUnknownSize or arg.maxAlign == szUnknownSize or arg.offset == szUnknownSize:
+    arg.maxAlign = szUnknownSize
+    arg.offset = szUnknownSize
+  else:
+    arg.maxAlign = max(value, arg.maxAlign)
+    arg.offset = align(arg.offset, value)
+
+proc finish(arg: var OffsetAccum) =
+  if arg.maxAlign == szUnknownSize or arg.offset == szUnknownSize:
+    arg.offset = szUnknownSize
+  else:
+    arg.offset = align(arg.offset, arg.maxAlign)
+
 proc computeSizeAlign(conf: ConfigRef; typ: PType)
 
 proc computeSubObjectAlign(conf: ConfigRef; n: PNode): BiggestInt =
@@ -49,6 +85,14 @@ proc computeSubObjectAlign(conf: ConfigRef; n: PNode): BiggestInt =
   else:
     result = 1
 
+
+proc setOffsetsToUnknown(n: PNode) =
+  if n.kind == nkSym and n.sym.kind == skField:
+    n.sym.offset = szUnknownSize
+  else:
+    for i in 0 ..< safeLen(n):
+      setOffsetsToUnknown(n[i])
+
 proc computeObjectOffsetsFoldFunction(conf: ConfigRef; n: PNode,
                                       initialOffset: BiggestInt): tuple[offset, align: BiggestInt] =
   ## ``offset`` is the offset within the object, after the node has been written, no padding bytes added
@@ -65,7 +109,7 @@ proc computeObjectOffsetsFoldFunction(conf: ConfigRef; n: PNode,
     assert(n.sons[0].kind == nkSym)
     let (kindOffset, kindAlign) = computeObjectOffsetsFoldFunction(conf, n.sons[0], initialOffset)
 
-    var maxChildAlign: BiggestInt = 0
+    var maxChildAlign: BiggestInt = if initialOffset == szUnknownSize: szUnknownSize else: 0
     for i in 1 ..< sonsLen(n):
       let child = n.sons[i]
       case child.kind
@@ -83,6 +127,7 @@ proc computeObjectOffsetsFoldFunction(conf: ConfigRef; n: PNode,
       else:
         internalError(conf, "computeObjectOffsetsFoldFunction(record case branch)")
     if maxChildAlign == szUnknownSize:
+      setOffsetsToUnknown(n)
       result.align  = szUnknownSize
       result.offset = szUnknownSize
     else:
@@ -145,9 +190,11 @@ proc computePackedObjectOffsetsFoldFunction(conf: ConfigRef; n: PNode, initialOf
     var maxChildOffset: BiggestInt = kindUnionOffset
     for i in 1 ..< sonsLen(n):
       let offset = computePackedObjectOffsetsFoldFunction(conf, n.sons[i].lastSon, kindUnionOffset, debug)
-      if offset < 0:
+      if offset == szIllegalRecursion:
-         result = offset
+        return szIllegalRecursion
-         break
+      if offset == szUnknownSize or maxChildOffset == szUnknownSize:
+        maxChildOffset = szUnknownSize
+      else:
         maxChildOffset = max(maxChildOffset, offset)
     result = maxChildOffset
   of nkRecList:
@@ -335,19 +382,22 @@ proc computeSizeAlign(conf: ConfigRef; typ: PType) =
     typ.size = typ.sons[0].size
     typ.align = typ.sons[0].align
   of tyTuple:
-    maxAlign = 1
+    try:
-    sizeAccum = 0
+      var accum = OffsetAccum(maxAlign: 1)
       for i in 0 ..< sonsLen(typ):
         let child = typ.sons[i]
         computeSizeAlign(conf, child)
-      if child.size < 0:
+        accum.align(child.align)
-        typ.size = child.size
+        if typ.n != nil: # is named tuple (has field symbols)?
-        typ.align = child.align
+          let sym = typ.n[i].sym
-        return
+          sym.offset = accum.offset
-      maxAlign = max(maxAlign, child.align)
+        accum.inc(int(child.size))
-      sizeAccum = align(sizeAccum, child.align) + child.size
+      accum.finish
-    typ.size = align(sizeAccum, maxAlign)
+      typ.size = accum.offset
-    typ.align = int16(maxAlign)
+      typ.align = int16(accum.maxAlign)
+    except IllegalTypeRecursionError:
+      typ.size = szIllegalRecursion
+      typ.align = szIllegalRecursion
   of tyObject:
     var headerSize: BiggestInt
     var headerAlign: int16
@@ -448,3 +498,58 @@ proc computeSizeAlign(conf: ConfigRef; typ: PType) =
   else:
     typ.size = szUncomputedSize
     typ.align = szUncomputedSize
+
+template foldSizeOf*(conf: ConfigRef; n: PNode; fallback: PNode): PNode =
+  let config = conf
+  let node = n
+  let typ = node[1].typ
+  computeSizeAlign(config, typ)
+  let size = typ.size
+  if size >= 0:
+    let res = newIntNode(nkIntLit, size)
+    res.info = node.info
+    res.typ = node.typ
+    res
+  else:
+    fallback
+
+template foldAlignOf*(conf: ConfigRef; n: PNode; fallback: PNode): PNode =
+  let config = conf
+  let node = n
+  let typ = node[1].typ
+  computeSizeAlign(config, typ)
+  let align = typ.align
+  if align >= 0:
+    let res = newIntNode(nkIntLit, align)
+    res.info = node.info
+    res.typ = node.typ
+    res
+  else:
+    fallback
+
+template foldOffsetOf*(conf: ConfigRef; n: PNode; fallback: PNode): PNode =
+  ## Returns an int literal node of the given offsetof expression in `n`.
+  ## Falls back to `fallback`, if the `offsetof` expression can't be processed.
+  let config = conf
+  let node : PNode = n
+  var dotExpr: PNode
+  block findDotExpr:
+    if node[1].kind == nkDotExpr:
+      dotExpr = node[1]
+    elif node[1].kind == nkCheckedFieldExpr:
+      dotExpr = node[1][0]
+    else:
+      localError(config, node.info, "can't compute offsetof on this ast")
+
+  assert dotExpr != nil
+  let value = dotExpr[0]
+  let member = dotExpr[1]
+  computeSizeAlign(config, value.typ)
+  let offset = member.sym.offset
+  if offset >= 0:
+    let tmp = newIntNode(nkIntLit, offset)
+    tmp.info = node.info
+    tmp.typ = node.typ
+    tmp
+  else:
+    fallback

compiler/vmgen.nim

@@ -1310,8 +1310,12 @@ proc genMagic(c: PCtx; n: PNode; dest: var TDest; m: TMagic) =
       # produces a value
     else:
       globalError(c.config, n.info, "expandToAst requires a call expression")
-  of mSizeOf, mAlignOf:
+  of mSizeOf:
-    globalError(c.config, n.info, "cannot evaluate 'sizeof/alignof' because its type is not defined completely")
+    globalError(c.config, n.info, "cannot evaluate 'sizeof' because its type is not defined completely")
+  of mAlignOf:
+    globalError(c.config, n.info, "cannot evaluate 'alignof' because its type is not defined completely")
+  of mOffsetOf:
+    globalError(c.config, n.info, "cannot evaluate 'offsetof' because its type is not defined completely")
   of mRunnableExamples:
     discard "just ignore any call to runnableExamples"
   of mDestroy: discard "ignore calls to the default destructor"

tests/misc/tsizeof.nim

@@ -549,3 +549,87 @@ proc payloadCheck() =
   doAssert sizeOf(Payload) == 4
 
 payloadCheck()
+
+# offsetof tuple types
+
+type
+  MyTupleType = tuple
+    a: float64
+    b: float64
+    c: float64
+
+  MyOtherTupleType = tuple
+    a: float64
+    b: imported_double
+    c: float64
+
+  MyCaseObject = object
+    val1: imported_double
+    case kind: bool
+    of true:
+      val2,val3: float32
+    else:
+      val4,val5: int32
+
+doAssert offsetof(MyTupleType, a) == 0
+doAssert offsetof(MyTupleType, b) == 8
+doAssert offsetof(MyTupleType, c) == 16
+
+doAssert offsetof(MyOtherTupleType, a) == 0
+doAssert offsetof(MyOtherTupleType, b) == 8
+
+# The following expression can only work if the offsetof expression is
+# properly forwarded for the C code generator.
+doAssert offsetof(MyOtherTupleType, c) == 16
+doAssert offsetof(Bar, foo) == 4
+doAssert offsetof(MyCaseObject, val1) == 0
+doAssert offsetof(MyCaseObject, kind) == 8
+doAssert offsetof(MyCaseObject, val2) == 12
+doAssert offsetof(MyCaseObject, val3) == 16
+doAssert offsetof(MyCaseObject, val4) == 12
+doAssert offsetof(MyCaseObject, val5) == 16
+
+template reject(e) =
+  static: assert(not compiles(e))
+
+reject:
+  const off1 = offsetof(MyOtherTupleType, c)
+
+reject:
+  const off2 = offsetof(MyOtherTupleType, b)
+
+reject:
+  const off3 = offsetof(MyCaseObject, kind)
+
+
+type
+  MyPackedCaseObject {.packed.} = object
+    val1: imported_double
+    case kind: bool
+    of true:
+      val2,val3: float32
+    else:
+      val4,val5: int32
+
+# packed case object
+
+doAssert offsetof(MyPackedCaseObject, val1) == 0
+doAssert offsetof(MyPackedCaseObject, val2) == 9
+doAssert offsetof(MyPackedCaseObject, val3) == 13
+doAssert offsetof(MyPackedCaseObject, val4) == 9
+doAssert offsetof(MyPackedCaseObject, val5) == 13
+
+reject:
+  const off4 = offsetof(MyPackedCaseObject, val1)
+
+reject:
+  const off5 = offsetof(MyPackedCaseObject, val2)
+
+reject:
+  const off6 = offsetof(MyPackedCaseObject, val3)
+
+reject:
+  const off7 = offsetof(MyPackedCaseObject, val4)
+
+reject:
+  const off8 = offsetof(MyPackedCaseObject, val5)

tests/misc/tsizeof2.nim

@@ -1,5 +1,5 @@
 discard """
-errormsg: "cannot evaluate 'sizeof/alignof' because its type is not defined completely"
+errormsg: "cannot evaluate 'sizeof' because its type is not defined completely"
 line: 9
 """