implement sizeof and alignof operator (manually squashed #5664) (#9356)

compiler/ast.nim

@@ -582,7 +582,8 @@ type
   TMagic* = enum # symbols that require compiler magic:
     mNone,
     mDefined, mDefinedInScope, mCompiles, mArrGet, mArrPut, mAsgn,
-    mLow, mHigh, mSizeOf, mTypeTrait, mIs, mOf, mAddr, mType, mTypeOf,
+    mLow, mHigh, mSizeOf, mAlignOf, mOffsetOf, mTypeTrait,
+    mIs, mOf, mAddr, mType, mTypeOf,
     mRoof, mPlugin, mEcho, mShallowCopy, mSlurp, mStaticExec, mStatic,
     mParseExprToAst, mParseStmtToAst, mExpandToAst, mQuoteAst,
     mUnaryLt, mInc, mDec, mOrd,
@@ -698,11 +699,6 @@ const
     mConStrStr, mAppendStrCh, mAppendStrStr, mAppendSeqElem,
     mInRange, mInSet, mRepr,
     mCopyStr, mCopyStrLast}
-  # magics that require special semantic checking and
-  # thus cannot be overloaded (also documented in the spec!):
-  SpecialSemMagics* = {
-    mDefined, mDefinedInScope, mCompiles, mLow, mHigh, mSizeOf, mIs, mOf,
-    mShallowCopy, mExpandToAst, mParallel, mSpawn, mAstToStr}
 
 type
   PNode* = ref TNode
@@ -1274,7 +1270,7 @@ proc newType*(kind: TTypeKind, owner: PSym): PType =
   result.kind = kind
   result.owner = owner
   result.size = -1
-  result.align = 2            # default alignment
+  result.align = -1            # default alignment
   result.id = getID()
   result.lockLevel = UnspecifiedLockLevel
   when debugIds:

compiler/canonicalizer.nim

@@ -300,7 +300,7 @@ proc encodeType(w: PRodWriter, t: PType, result: var string) =
   if t.size != - 1:
     add(result, '/')
     encodeVBiggestInt(t.size, result)
-  if t.align != 2:
+  if t.align != - 1:
     add(result, '=')
     encodeVInt(t.align, result)
   encodeLoc(w, t.loc, result)

compiler/condsyms.nim

@@ -58,6 +58,7 @@ proc initDefines*(symbols: StringTableRef) =
   defineSymbol("nimtypedescfixed")
   defineSymbol("nimKnowsNimvm")
   defineSymbol("nimArrIdx")
+  defineSymbol("nimHasalignOf")
   defineSymbol("nimImmediateDeprecated")
   defineSymbol("nimNewShiftOps")
   defineSymbol("nimDistros")

compiler/jsgen.nim

@@ -1759,7 +1759,6 @@ proc genMagic(p: PProc, n: PNode, r: var TCompRes) =
   of mIsNil: unaryExpr(p, n, r, "", "($1 === null)")
   of mEnumToStr: genRepr(p, n, r)
   of mNew, mNewFinalize: genNew(p, n)
-  of mSizeOf: r.res = rope(getSize(p.config, n.sons[1].typ))
   of mChr, mArrToSeq: gen(p, n.sons[1], r)      # nothing to do
   of mOrd: genOrd(p, n, r)
   of mLengthStr, mLengthSeq, mLengthOpenArray, mLengthArray:
@@ -2356,4 +2355,3 @@ proc myOpen(graph: ModuleGraph; s: PSym): PPassContext =
   result = newModule(graph, s)
 
 const JSgenPass* = makePass(myOpen, myProcess, myClose)
-

compiler/pragmas.nim

@@ -803,9 +803,11 @@ proc singlePragma(c: PContext, sym: PSym, n: PNode, i: var int,
         if sym.typ == nil: invalidPragma(c, it)
         var size = expectIntLit(c, it)
         if not isPowerOfTwo(size) or size <= 0 or size > 8:
-          localError(c.config, it.info, "power of two expected")
+          localError(c.config, it.info, "size may only be 1, 2, 4 or 8")
         else:
           sym.typ.size = size
+          # TODO, this is not correct
+          sym.typ.align = int16(size)
       of wNodecl:
         noVal(c, it)
         incl(sym.loc.flags, lfNoDecl)

compiler/semexprs.nim

@@ -158,6 +158,10 @@ proc isCastable(conf: ConfigRef; dst, src: PType): bool =
   var dstSize, srcSize: BiggestInt
   dstSize = computeSize(conf, dst)
   srcSize = computeSize(conf, src)
+  if dstSize == -3 or srcSize == -3: # szUnknownSize
+    # The Nim compiler can't detect if it's legal or not.
+    # Just assume the programmer knows what he is doing.
+    return true
   if dstSize < 0:
     result = false
   elif srcSize < 0:
@@ -308,15 +312,6 @@ proc semLowHigh(c: PContext, n: PNode, m: TMagic): PNode =
       localError(c.config, n.info, "invalid argument for: " & opToStr[m])
   result = n
 
-proc semSizeof(c: PContext, n: PNode): PNode =
-  if sonsLen(n) != 2:
-    localError(c.config, n.info, errXExpectsTypeOrValue % "sizeof")
-  else:
-    n.sons[1] = semExprWithType(c, n.sons[1], {efDetermineType})
-    #restoreOldStyleType(n.sons[1])
-  n.typ = getSysType(c.graph, n.info, tyInt)
-  result = n
-
 proc fixupStaticType(c: PContext, n: PNode) =
   # This proc can be applied to evaluated expressions to assign
   # them a static type.
@@ -1958,7 +1953,6 @@ proc setMs(n: PNode, s: PSym): PNode =
 
 proc semMagic(c: PContext, n: PNode, s: PSym, flags: TExprFlags): PNode =
   # this is a hotspot in the compiler!
-  # DON'T forget to update ast.SpecialSemMagics if you add a magic here!
   result = n
   case s.magic # magics that need special treatment
   of mAddr:
@@ -1975,7 +1969,6 @@ proc semMagic(c: PContext, n: PNode, s: PSym, flags: TExprFlags): PNode =
   of mCompiles: result = semCompiles(c, setMs(n, s), flags)
   #of mLow: result = semLowHigh(c, setMs(n, s), mLow)
   #of mHigh: result = semLowHigh(c, setMs(n, s), mHigh)
-  of mSizeOf: result = semSizeof(c, setMs(n, s))
   of mIs: result = semIs(c, setMs(n, s), flags)
   #of mOf: result = semOf(c, setMs(n, s))
   of mShallowCopy: result = semShallowCopy(c, n, flags)

compiler/semfold.nim

@@ -646,18 +646,6 @@ proc getConstExpr(m: PSym, n: PNode; g: ModuleGraph): PNode =
       of mNone:
         # If it has no sideEffect, it should be evaluated. But not here.
         return
-      of mSizeOf:
-        var a = n.sons[1]
-        if computeSize(g.config, a.typ) < 0:
-          localError(g.config, a.info, "cannot evaluate 'sizeof' because its type is not defined completely")
-          result = nil
-        elif skipTypes(a.typ, typedescInst+{tyRange, tyArray}).kind in
-             IntegralTypes+NilableTypes+{tySet}:
-          #{tyArray,tyObject,tyTuple}:
-          result = newIntNodeT(getSize(g.config, a.typ), n, g)
-        else:
-          result = nil
-          # XXX: size computation for complex types is still wrong
       of mLow:
         result = newIntNodeT(firstOrd(g.config, n.sons[1].typ), n, g)
       of mHigh:

compiler/semmagic.nim

@@ -307,6 +307,14 @@ proc semOf(c: PContext, n: PNode): PNode =
 
 proc magicsAfterOverloadResolution(c: PContext, n: PNode,
                                    flags: TExprFlags): PNode =
+  ## This is the preferred code point to implement magics.
+  ## This function basically works like a macro, with the difference
+  ## that it is implemented in the compiler and not on the nimvm.
+  ## ``c`` the current module, a symbol table to a very good approximation
+  ## ``n`` the ast like it would be passed to a real macro
+  ## ``flags`` Some flags for more contextual information on how the
+  ## "macro" is calld.
+
   case n[0].sym.magic
   of mAddr:
     checkSonsLen(n, 2, c.config)
@@ -314,8 +322,53 @@ proc magicsAfterOverloadResolution(c: PContext, n: PNode,
   of mTypeOf:
     checkSonsLen(n, 2, c.config)
     result = semTypeOf(c, n.sons[1])
-  of mArrGet: result = semArrGet(c, n, flags)
+  of mSizeOf:
-  of mArrPut: result = semArrPut(c, n, flags)
+      # TODO there is no proper way to find out if a type cannot be queried for the size.
+      let size = getSize(c.config, n[1].typ)
+      # We just assume here that the type might come from the c backend
+      if size == szUnknownSize:
+        # Forward to the c code generation to emit a `sizeof` in the C code.
+        result = n
+      elif size >= 0:
+        result = newIntNode(nkIntLit, size)
+        result.info = n.info
+        result.typ = n.typ
+      else:
+
+        localError(c.config, n.info, "cannot evaluate 'sizeof' because its type is not defined completely")
+
+        result = nil
+
+
+  of mAlignOf:
+    result = newIntNode(nkIntLit, getAlign(c.config, n[1].typ))
+    result.info = n.info
+    result.typ = n.typ
+  of mOffsetOf:
+    var dotExpr: PNode
+
+    block findDotExpr:
+      if n[1].kind == nkDotExpr:
+        dotExpr = n[1]
+      elif n[1].kind == nkCheckedFieldExpr:
+        dotExpr = n[1][0]
+      else:
+        illFormedAst(n, c.config)
+
+    assert dotExpr != nil
+
+    let value = dotExpr[0]
+    let member = dotExpr[1]
+
+    discard computeSize(c.config, value.typ)
+
+    result = newIntNode(nkIntLit, member.sym.offset)
+    result.info = n.info
+    result.typ = n.typ
+  of mArrGet:
+    result = semArrGet(c, n, flags)
+  of mArrPut:
+    result = semArrPut(c, n, flags)
   of mAsgn:
     if n[0].sym.name.s == "=":
       result = semAsgnOpr(c, n)

compiler/semtypes.nim

@@ -1848,4 +1848,3 @@ proc semGenericParamList(c: PContext, n: PNode, father: PType = nil): PNode =
       s.position = result.len
       addSon(result, newSymNode(s))
       if sfGenSym notin s.flags: addDecl(c, s)
-

compiler/sigmatch.nim

@@ -2612,4 +2612,3 @@ tests:
 
     yes int, ordinal
     no  string, ordinal
-

compiler/sizealignoffsetimpl.nim

@@ -0,0 +1,447 @@
+
+proc align(address, alignment: BiggestInt): BiggestInt =
+  result = (address + (alignment - 1)) and not (alignment - 1)
+
+const
+  ## a size is concidered "unknown" when it is an imported type from C
+  ## or C++.
+  szUnknownSize*      = -3
+  szIllegalRecursion* = -2
+  szUncomputedSize*   = -1
+
+proc computeSizeAlign(conf: ConfigRef; typ: PType): void
+
+proc computeSubObjectAlign(conf: ConfigRef; n: PNode): BiggestInt =
+  ## returns object alignment
+  case n.kind
+  of nkRecCase:
+    assert(n.sons[0].kind == nkSym)
+    result = computeSubObjectAlign(conf, n.sons[0])
+
+    for i in 1 ..< sonsLen(n):
+      let child = n.sons[i]
+      case child.kind
+      of nkOfBranch, nkElse:
+        let align = computeSubObjectAlign(conf, child.lastSon)
+        if align < 0:
+          return align
+        result = max(result, align)
+      else:
+        internalError(conf, "computeSubObjectAlign")
+
+  of nkRecList:
+    result = 1
+
+    for i, child in n.sons:
+      let align = computeSubObjectAlign(conf, n.sons[i])
+      if align < 0:
+        return align
+
+      result = max(result, align)
+
+  of nkSym:
+    computeSizeAlign(conf, n.sym.typ)
+    result = n.sym.typ.align
+
+  else:
+    result = 1
+
+proc computeObjectOffsetsFoldFunction(conf: ConfigRef; n: PNode, initialOffset: BiggestInt): tuple[offset, align: BiggestInt] =
+  ## ``offset`` is the offset within the object, after the node has been written, no padding bytes added
+  ## ``align`` maximum alignment from all sub nodes
+
+  if n.typ != nil and n.typ.size == szIllegalRecursion:
+    result.offset = szIllegalRecursion
+    result.align  = szIllegalRecursion
+    return
+
+  result.align = 1
+  case n.kind
+  of nkRecCase:
+
+    assert(n.sons[0].kind == nkSym)
+    let (kindOffset, kindAlign) = computeObjectOffsetsFoldFunction(conf, n.sons[0], initialOffset)
+
+    var maxChildAlign: BiggestInt = 0
+
+    for i in 1 ..< sonsLen(n):
+      let child = n.sons[i]
+      case child.kind
+      of nkOfBranch, nkElse:
+        # offset parameter cannot be known yet, it needs to know the alignment first
+        let align = computeSubObjectAlign(conf, n.sons[i].lastSon)
+
+        if align == szIllegalRecursion:
+          result.offset  = szIllegalRecursion
+          result.align = szIllegalRecursion
+          return
+
+        if align == szUnknownSize or maxChildAlign == szUnknownSize:
+          maxChildAlign = szUnknownSize
+        else:
+          maxChildAlign = max(maxChildAlign, align)
+      else:
+        internalError(conf, "computeObjectOffsetsFoldFunction(record case branch)")
+
+    if maxChildAlign == szUnknownSize:
+      result.align  = szUnknownSize
+      result.offset = szUnknownSize
+
+    else:
+      # the union neds to be aligned first, before the offsets can be assigned
+      let kindUnionOffset = align(kindOffset, maxChildAlign)
+
+      var maxChildOffset: BiggestInt = 0
+      for i in 1 ..< sonsLen(n):
+        let (offset, align) = computeObjectOffsetsFoldFunction(conf, n.sons[i].lastSon, kindUnionOffset)
+        maxChildOffset = max(maxChildOffset, offset)
+
+      result.align = max(kindAlign, maxChildAlign)
+      result.offset  = maxChildOffset
+
+
+  of nkRecList:
+
+    result.align = 1 # maximum of all member alignments
+
+    var offset = initialOffset
+
+    for i, child in n.sons:
+      let (new_offset, align) = computeObjectOffsetsFoldFunction(conf, child, offset)
+
+      if new_offset == szIllegalRecursion:
+        result.offset  = szIllegalRecursion
+        result.align = szIllegalRecursion
+        return
+
+      elif new_offset == szUnknownSize or offset == szUnknownSize:
+        # if anything is unknown, the rest becomes unknown as well
+        offset = szUnknownSize
+        result.align  = szUnknownSize
+
+      else:
+        offset = new_offset
+        result.align = max(result.align, align)
+
+    # final alignment
+    if offset == szUnknownSize:
+      result.offset = szUnknownSize
+    else:
+      result.offset = align(offset, result.align)
+
+  of nkSym:
+    computeSizeAlign(conf, n.sym.typ)
+    let size  = n.sym.typ.size
+    let align = n.sym.typ.align
+    result.align  = align
+    if initialOffset == szUnknownSize:
+      n.sym.offset  = szUnknownSize
+      result.offset = szUnknownSize
+    else:
+      n.sym.offset = align(initialOffset, align).int
+      result.offset = n.sym.offset + n.sym.typ.size
+
+  else:
+    result.align  = szUnknownSize
+    result.offset = szUnknownSize
+
+
+var recDepth = 0
+proc computePackedObjectOffsetsFoldFunction(conf: ConfigRef; n: PNode, initialOffset: BiggestInt, debug : bool): BiggestInt =
+  ## ``result`` is the offset within the object, after the node has been written, no padding bytes added
+  recDepth += 1
+  defer:
+    recDepth -= 1
+
+  if debug:
+    if n.kind == nkSym:
+      echo repeat("--", recDepth) & "> ", initialOffset, "  ", n.kind, "  ", n.sym.name.s
+    else:
+      echo repeat("--", recDepth) & "> ", initialOffset, "  ", n.kind
+
+  case n.kind
+  of nkRecCase:
+
+    assert(n.sons[0].kind == nkSym)
+    let kindOffset = computePackedObjectOffsetsFoldFunction(conf, n.sons[0], initialOffset, debug)
+    # the union neds to be aligned first, before the offsets can be assigned
+    let kindUnionOffset = kindOffset
+
+    var maxChildOffset: BiggestInt = kindUnionOffset
+    for i in 1 ..< sonsLen(n):
+      let offset = computePackedObjectOffsetsFoldFunction(conf, n.sons[i].lastSon, kindUnionOffset, debug)
+      maxChildOffset = max(maxChildOffset, offset)
+
+    if debug:
+      echo repeat("  ", recDepth), "result: ", maxChildOffset
+
+    result  = maxChildOffset
+
+  of nkRecList:
+    result = initialOffset
+    for i, child in n.sons:
+      result = computePackedObjectOffsetsFoldFunction(conf, child, result, debug)
+      if result == szIllegalRecursion:
+        break
+
+  of nkSym:
+    computeSizeAlign(conf, n.sym.typ)
+    n.sym.offset = initialOffset.int
+    result = n.sym.offset + n.sym.typ.size
+
+  else:
+    result = szUnknownSize
+
+# TODO this one needs an alignment map of the individual types
+
+proc computeSizeAlign(conf: ConfigRef; typ: PType) =
+  ## computes and sets ``size`` and ``align`` members of ``typ``
+
+  let hasSize  = typ.size  != szUncomputedSize
+  let hasAlign = typ.align != szUncomputedSize
+
+  if hasSize and hasAlign:
+    # nothing to do, size and align already computed
+    return
+
+  # This function can only calculate both, size and align at the same time.
+  # If one of them is already set this value is stored here and reapplied
+  let revertSize = typ.size
+  let revertAlign = typ.align
+  defer:
+    if hasSize:
+      typ.size = revertSize
+    if hasAlign:
+      typ.align = revertAlign
+
+  if typ.size == szIllegalRecursion or typ.align == szIllegalRecursion:
+    # we are already computing the size of the type
+    # --> illegal recursion in type
+    return
+
+  # mark computation in progress
+  typ.size  = szIllegalRecursion
+  typ.align = szIllegalRecursion
+
+  var maxAlign, sizeAccum, length: BiggestInt
+
+  var tk = typ.kind
+  case tk
+  of tyProc:
+    if typ.callConv == ccClosure:
+      typ.size = 2 * conf.target.ptrSize
+    else:
+      typ.size = conf.target.ptrSize
+    typ.align = int16(conf.target.ptrSize)
+
+  of tyNil:
+    typ.size = conf.target.ptrSize
+    typ.align = int16(conf.target.ptrSize)
+
+  of tyString:
+    if tfHasAsgn in typ.flags:
+      typ.size = conf.target.ptrSize * 2
+    else:
+      typ.size = conf.target.ptrSize
+    typ.align = int16(conf.target.ptrSize)
+
+  of tyCString, tySequence, tyPtr, tyRef, tyVar, tyLent, tyOpenArray:
+    let base = typ.lastSon
+    if base == typ:
+      # this is not the correct location to detect ``type A = ptr A``
+      typ.size  = szIllegalRecursion
+      typ.align = szIllegalRecursion
+      return
+
+    # recursive tuplers are not allowed and should be detected in the frontend
+    if base.kind == tyTuple:
+      computeSizeAlign(conf, base)
+      if base.size == szIllegalRecursion:
+        typ.size  = szIllegalRecursion
+        typ.align = szIllegalRecursion
+        return
+
+    typ.align = int16(conf.target.ptrSize)
+    if typ.kind == tySequence and tfHasAsgn in typ.flags:
+      typ.size  = conf.target.ptrSize * 2
+    else:
+      typ.size  = conf.target.ptrSize
+
+  of tyArray:
+    computeSizeAlign(conf, typ.sons[1])
+    let elemSize = typ.sons[1].size
+    if elemSize < 0:
+      typ.size  = elemSize
+      typ.align = int16(elemSize)
+    else:
+      typ.size  = lengthOrd(conf, typ.sons[0]) * elemSize
+      typ.align = typ.sons[1].align
+
+  of tyUncheckedArray:
+    let base = typ.lastSon
+    computeSizeAlign(conf, base)
+    # this should probably be szUnknownSize
+    typ.size = 0
+    typ.align = base.align
+  of tyEnum:
+    if firstOrd(conf, typ) < 0:
+      typ.size  = 4              # use signed int32
+      typ.align = 4
+    else:
+      length = lastOrd(conf, typ)   # BUGFIX: use lastOrd!
+      if length + 1 < `shl`(1, 8):
+        typ.size  = 1
+        typ.align = 1
+      elif length + 1 < `shl`(1, 16):
+        typ.size = 2
+        typ.align = 2
+      elif length + 1 < `shl`(BiggestInt(1), 32):
+        typ.size = 4
+        typ.align = 4
+      else:
+        typ.size = 8
+        typ.align = 8
+
+  of tySet:
+    if typ.sons[0].kind == tyGenericParam:
+      typ.size  = szUncomputedSize
+      typ.align = szUncomputedSize # in original version this was 1
+    else:
+      length = lengthOrd(conf, typ.sons[0])
+      if length <= 8:
+        typ.size = 1
+      elif length <= 16:
+        typ.size = 2
+      elif length <= 32:
+        typ.size = 4
+      elif length <= 64:
+        typ.size = 8
+      elif align(length, 8) mod 8 == 0:
+        typ.size  = align(length, 8) div 8
+      else:
+        typ.size = align(length, 8) div 8 + 1
+    typ.align = int16(typ.size)
+
+  of tyRange:
+    computeSizeAlign(conf, typ.sons[0])
+    typ.size = typ.sons[0].size
+    typ.align = typ.sons[0].align
+
+  of tyTuple:
+    maxAlign = 1
+    sizeAccum = 0
+
+    for i in countup(0, sonsLen(typ) - 1):
+      let child = typ.sons[i]
+      computeSizeAlign(conf, child)
+
+      if child.size == szIllegalRecursion:
+        typ.size  = szIllegalRecursion
+        typ.align = szIllegalRecursion
+        return
+
+      maxAlign = max(maxAlign, child.align)
+      sizeAccum = align(sizeAccum, child.align) + child.size
+
+    typ.size  = align(sizeAccum, maxAlign)
+    typ.align = int16(maxAlign)
+
+  of tyObject:
+    var headerSize : BiggestInt
+    var headerAlign: int16
+
+    if typ.sons[0] != nil:
+      # compute header size
+      var st = typ.sons[0]
+
+      while st.kind in skipPtrs:
+        st = st.sons[^1]
+
+      computeSizeAlign(conf, st)
+      if st.size == szIllegalRecursion:
+        typ.size = st.size
+        typ.align = st.align
+        return
+
+      headerSize  = st.size
+      headerAlign = st.align
+
+    elif isObjectWithTypeFieldPredicate(typ):
+      # this branch is taken for RootObj
+      headerSize = conf.target.intSize
+      headerAlign = conf.target.intSize.int16
+
+    else:
+      headerSize  = 0
+      headerAlign = 1
+
+    let (offset, align) =
+      if tfPacked in typ.flags:
+        (computePackedObjectOffsetsFoldFunction(conf, typ.n, headerSize, false), BiggestInt(1))
+      else:
+        computeObjectOffsetsFoldFunction(conf, typ.n, headerSize)
+
+    if offset == szIllegalRecursion:
+      typ.size = szIllegalRecursion
+      typ.align = szIllegalRecursion
+      return
+
+    if offset == szUnknownSize or (
+          typ.sym != nil and
+          typ.sym.flags * {sfCompilerProc, sfImportc} == {sfImportc}
+    ):
+      typ.size  = szUnknownSize
+      typ.align = szUnknownSize
+      return
+
+    # header size is already in size from computeObjectOffsetsFoldFunction
+    # maxAlign is probably not changed at all from headerAlign
+
+    if tfPacked in typ.flags:
+      typ.size = offset
+      typ.align = 1
+    else:
+      typ.align = int16(max(align, headerAlign))
+      typ.size  = align(offset, typ.align)
+
+  of tyInferred:
+    if typ.len > 1:
+      computeSizeAlign(conf, typ.lastSon)
+      typ.size = typ.lastSon.size
+      typ.align = typ.lastSon.align
+
+  of tyGenericInst, tyDistinct, tyGenericBody, tyAlias:
+    computeSizeAlign(conf, typ.lastSon)
+    typ.size = typ.lastSon.size
+    typ.align = typ.lastSon.align
+
+  of tyTypeClasses:
+    if typ.isResolvedUserTypeClass:
+      computeSizeAlign(conf, typ.lastSon)
+      typ.size = typ.lastSon.size
+      typ.align = typ.lastSon.align
+    else:
+      typ.size = szUncomputedSize
+      typ.align = szUncomputedSize
+
+  of tyTypeDesc:
+    computeSizeAlign(conf, typ.base)
+    typ.size = typ.base.size
+    typ.align = typ.base.align
+
+  of tyForward:
+    # is this really illegal recursion, or maybe just unknown?
+    typ.size = szIllegalRecursion
+    typ.align = szIllegalRecursion
+
+  of tyStatic:
+    if typ.n != nil:
+      computeSizeAlign(conf, typ.lastSon)
+      typ.size = typ.lastSon.size
+      typ.align = typ.lastSon.align
+    else:
+      typ.size = szUncomputedSize
+      typ.align = szUncomputedSize
+  else:
+    typ.size  = szUncomputedSize
+    typ.align = szUncomputedSize

compiler/types.nim

@@ -238,6 +238,7 @@ proc containsObject*(t: PType): bool =
   result = searchTypeFor(t, isObjectPredicate)
 
 proc isObjectWithTypeFieldPredicate(t: PType): bool =
+
   result = t.kind == tyObject and t.sons[0] == nil and
       not (t.sym != nil and {sfPure, sfInfixCall} * t.sym.flags != {}) and
       tfFinal notin t.flags
@@ -1142,7 +1143,6 @@ proc typeAllowedNode(marker: var IntSet, n: PNode, kind: TSymKind,
                      flags: TTypeAllowedFlags = {}): PType =
   if n != nil:
     result = typeAllowedAux(marker, n.typ, kind, flags)
-    #if not result: debug(n.typ)
     if result == nil:
       case n.kind
       of nkNone..nkNilLit:
@@ -1266,196 +1266,24 @@ proc typeAllowed*(t: PType, kind: TSymKind; flags: TTypeAllowedFlags = {}): PTyp
   var marker = initIntSet()
   result = typeAllowedAux(marker, t, kind, flags)
 
-proc align(address, alignment: BiggestInt): BiggestInt =
+include sizealignoffsetimpl
-  result = (address + (alignment - 1)) and not (alignment - 1)
-
-const
-  szNonConcreteType* = -3
-  szIllegalRecursion* = -2
-  szUnknownSize* = -1
-
-proc computeSizeAux(conf: ConfigRef; typ: PType, a: var BiggestInt): BiggestInt
-proc computeRecSizeAux(conf: ConfigRef; n: PNode, a, currOffset: var BiggestInt): BiggestInt =
-  var maxAlign, maxSize, b, res: BiggestInt
-  case n.kind
-  of nkRecCase:
-    assert(n.sons[0].kind == nkSym)
-    result = computeRecSizeAux(conf, n.sons[0], a, currOffset)
-    maxSize = 0
-    maxAlign = 1
-    for i in countup(1, sonsLen(n) - 1):
-      case n.sons[i].kind
-      of nkOfBranch, nkElse:
-        res = computeRecSizeAux(conf, lastSon(n.sons[i]), b, currOffset)
-        if res < 0: return res
-        maxSize = max(maxSize, res)
-        maxAlign = max(maxAlign, b)
-      else:
-        return szIllegalRecursion
-    currOffset = align(currOffset, maxAlign) + maxSize
-    result = align(result, maxAlign) + maxSize
-    a = maxAlign
-  of nkRecList:
-    result = 0
-    maxAlign = 1
-    for i in countup(0, sonsLen(n) - 1):
-      res = computeRecSizeAux(conf, n.sons[i], b, currOffset)
-      if res < 0: return res
-      currOffset = align(currOffset, b) + res
-      result = align(result, b) + res
-      if b > maxAlign: maxAlign = b
-    a = maxAlign
-  of nkSym:
-    result = computeSizeAux(conf, n.sym.typ, a)
-    n.sym.offset = int(currOffset)
-  else:
-    a = 1
-    result = szNonConcreteType
-
-proc computeSizeAux(conf: ConfigRef; typ: PType, a: var BiggestInt): BiggestInt =
-  var res, maxAlign, length, currOffset: BiggestInt
-  if typ.size == szIllegalRecursion:
-    # we are already computing the size of the type
-    # --> illegal recursion in type
-    return szIllegalRecursion
-  if typ.size >= 0:
-    # size already computed
-    result = typ.size
-    a = typ.align
-    return
-  typ.size = szIllegalRecursion # mark as being computed
-  case typ.kind
-  of tyInt, tyUInt:
-    result = conf.target.intSize
-    a = result
-  of tyInt8, tyUInt8, tyBool, tyChar:
-    result = 1
-    a = result
-  of tyInt16, tyUInt16:
-    result = 2
-    a = result
-  of tyInt32, tyUInt32, tyFloat32:
-    result = 4
-    a = result
-  of tyInt64, tyUInt64, tyFloat64:
-    result = 8
-    a = result
-  of tyFloat128:
-    result = 16
-    a = result
-  of tyFloat:
-    result = conf.target.floatSize
-    a = result
-  of tyProc:
-    if typ.callConv == ccClosure: result = 2 * conf.target.ptrSize
-    else: result = conf.target.ptrSize
-    a = conf.target.ptrSize
-  of tyString:
-    if tfHasAsgn in typ.flags:
-      result = conf.target.ptrSize * 2
-    else:
-      result = conf.target.ptrSize
-  of tyNil:
-    result = conf.target.ptrSize
-    a = result
-  of tyCString, tySequence, tyPtr, tyRef, tyVar, tyLent, tyOpenArray:
-    let base = typ.lastSon
-    if base == typ or (base.kind == tyTuple and base.size==szIllegalRecursion):
-      result = szIllegalRecursion
-    else:
-      if typ.kind == tySequence and tfHasAsgn in typ.flags:
-        result = conf.target.ptrSize * 2
-      else:
-        result = conf.target.ptrSize
-    a = result
-  of tyArray:
-    let elemSize = computeSizeAux(conf, typ.sons[1], a)
-    if elemSize < 0: return elemSize
-    result = lengthOrd(conf, typ.sons[0]) * elemSize
-  of tyEnum:
-    if firstOrd(conf, typ) < 0:
-      result = 4              # use signed int32
-    else:
-      length = lastOrd(conf, typ)   # BUGFIX: use lastOrd!
-      if length + 1 < `shl`(1, 8): result = 1
-      elif length + 1 < `shl`(1, 16): result = 2
-      elif length + 1 < `shl`(BiggestInt(1), 32): result = 4
-      else: result = 8
-    a = result
-  of tySet:
-    if typ.sons[0].kind == tyGenericParam:
-      result = szUnknownSize
-    else:
-      length = lengthOrd(conf, typ.sons[0])
-      if length <= 8: result = 1
-      elif length <= 16: result = 2
-      elif length <= 32: result = 4
-      elif length <= 64: result = 8
-      elif align(length, 8) mod 8 == 0: result = align(length, 8) div 8
-      else: result = align(length, 8) div 8 + 1
-    a = result
-  of tyRange:
-    result = computeSizeAux(conf, typ.sons[0], a)
-  of tyTuple:
-    result = 0
-    maxAlign = 1
-    for i in countup(0, sonsLen(typ) - 1):
-      res = computeSizeAux(conf, typ.sons[i], a)
-      if res < 0: return res
-      maxAlign = max(maxAlign, a)
-      result = align(result, a) + res
-    result = align(result, maxAlign)
-    a = maxAlign
-  of tyObject:
-    if typ.sons[0] != nil:
-      result = computeSizeAux(conf, typ.sons[0].skipTypes(skipPtrs), a)
-      if result < 0: return
-      maxAlign = a
-    elif isObjectWithTypeFieldPredicate(typ):
-      result = conf.target.intSize
-      maxAlign = result
-    else:
-      result = 0
-      maxAlign = 1
-    currOffset = result
-    result = computeRecSizeAux(conf, typ.n, a, currOffset)
-    if result < 0: return
-    if a < maxAlign: a = maxAlign
-    result = align(result, a)
-  of tyInferred:
-    if typ.len > 1:
-      result = computeSizeAux(conf, typ.lastSon, a)
-  of tyGenericInst, tyDistinct, tyGenericBody, tyAlias, tySink:
-    result = computeSizeAux(conf, lastSon(typ), a)
-  of tyTypeClasses:
-    result = if typ.isResolvedUserTypeClass: computeSizeAux(conf, typ.lastSon, a)
-             else: szUnknownSize
-  of tyTypeDesc:
-    result = computeSizeAux(conf, typ.base, a)
-  of tyForward: return szIllegalRecursion
-  of tyStatic:
-    result = if typ.n != nil: computeSizeAux(conf, typ.lastSon, a)
-             else: szUnknownSize
-  of tyUncheckedArray:
-    result = 0
-  else:
-    #internalError("computeSizeAux()")
-    result = szUnknownSize
-  typ.size = result
-  typ.align = int16(a)
 
 proc computeSize*(conf: ConfigRef; typ: PType): BiggestInt =
-  var a: BiggestInt = 1
+  computeSizeAlign(conf, typ)
-  result = computeSizeAux(conf, typ, a)
+  result = typ.size
 
 proc getReturnType*(s: PSym): PType =
   # Obtains the return type of a iterator/proc/macro/template
   assert s.kind in skProcKinds
   result = s.typ.sons[0]
 
+proc getAlign*(conf: ConfigRef; typ: PType): BiggestInt =
+  computeSizeAlign(conf, typ)
+  result = typ.align
+
 proc getSize*(conf: ConfigRef; typ: PType): BiggestInt =
-  result = computeSize(conf, typ)
+  computeSizeAlign(conf, typ)
-  if result < 0: internalError(conf, "getSize: " & $typ.kind)
+  result = typ.size
 
 proc containsGenericTypeIter(t: PType, closure: RootRef): bool =
   case t.kind

compiler/vmgen.nim

@@ -1080,8 +1080,6 @@ proc genMagic(c: PCtx; n: PNode; dest: var TDest; m: TMagic) =
     c.gABC(n, if m == mOf: opcOf else: opcIs, dest, tmp, idx)
     c.freeTemp(tmp)
     c.freeTemp(idx)
-  of mSizeOf:
-    globalError(c.config, n.info, "cannot run in the VM: " & renderTree(n))
   of mHigh:
     if dest < 0: dest = c.getTemp(n.typ)
     let tmp = c.genx(n.sons[1])
@@ -1231,6 +1229,8 @@ proc genMagic(c: PCtx; n: PNode; dest: var TDest; m: TMagic) =
       # produces a value
     else:
       globalError(c.config, n.info, "expandToAst requires a call expression")
+  of mSizeOf, mAlignOf:
+    globalError(c.config, n.info, "cannot evaluate 'sizeof/alignof' because its type is not defined completely")
   of mRunnableExamples:
     discard "just ignore any call to runnableExamples"
   else:

lib/system.nim

@@ -682,13 +682,30 @@ proc sizeof*[T](x: T): int {.magic: "SizeOf", noSideEffect.}
   ## that one never needs to know ``x``'s size. As a special semantic rule,
   ## ``x`` may also be a type identifier (``sizeof(int)`` is valid).
   ##
-  ## Limitations: If used within nim VM context ``sizeof`` will only work
+  ## Limitations: If used for types that are imported from C or C++,
-  ## for simple types.
+  ## sizeof should fallback to the ``sizeof`` in the C compiler. The
+  ## result isn't available for the Nim compiler and therefore can't
+  ## be used inside of macros.
   ##
   ## .. code-block:: nim
   ##  sizeof('A') #=> 1
   ##  sizeof(2) #=> 8
 
+when defined(nimHasalignOf):
+  proc alignof*[T](x: T): int {.magic: "AlignOf", noSideEffect.}
+  proc alignof*(x: typedesc): int {.magic: "AlignOf", noSideEffect.}
+
+  proc offsetOfDotExpr(typeAccess: typed): int {.magic: "OffsetOf", noSideEffect, compileTime.}
+
+  template offsetOf*[T](t: typedesc[T]; member: untyped): int =
+    var tmp: T
+    offsetOfDotExpr(tmp.member)
+
+  template offsetOf*[T](value: T; member: untyped): int =
+    offsetOfDotExpr(value.member)
+
+  #proc offsetOf*(memberaccess: typed): int {.magic: "OffsetOf", noSideEffect.}
+
 when defined(nimtypedescfixed):
   proc sizeof*(x: typedesc): int {.magic: "SizeOf", noSideEffect.}
 

testament/tester.nim

@@ -467,7 +467,7 @@ proc main() =
     of "targets":
       targetsStr = p.val.string
       targets = parseTargets(targetsStr)
-    of "nim": compilerPrefix = p.val.string
+    of "nim": compilerPrefix = p.val.string & " "
     else: quit Usage
     p.next()
   if p.kind != cmdArgument: quit Usage

tests/misc/tsizeof.nim

@@ -1,14 +1,8 @@
 discard """
-  file: "tsize.nim"
+  output: "OK"
-  output: "40 3 12 32"
 """
-type
-  TMyRecord {.final.} = object
-    x, y: int
-    b: bool
-    r: float
-    s: string
 
+type
   TMyEnum = enum
     tmOne, tmTwo, tmThree, tmFour
 
@@ -21,8 +15,340 @@ const
   mysize2 = sizeof(TMyArray2)
   mysize3 = sizeof(TMyArray3)
 
-write(stdout, sizeof(TMyRecord))
+assert mysize1 == 3
-echo ' ', mysize1, ' ', mysize2, ' ',mysize3
+assert mysize2 == 12
+assert mysize3 == 32
+
+import macros, typetraits
+
+macro testSizeAlignOf(args: varargs[untyped]): untyped =
+  result = newStmtList()
+  for arg in args:
+    result.add quote do:
+      let
+        c_size = c_sizeof(`arg`)
+        nim_size = sizeof(`arg`)
+        c_align = c_alignof(type(`arg`))
+        nim_align = alignof(`arg`)
+
+      if nim_size != c_size or nim_align != c_align:
+        var msg = strAlign(`arg`.type.name & ": ")
+        if nim_size != c_size:
+          msg.add  " size(got, expected):  " & $nim_size  & " != " & $c_size
+        if nim_align != c_align:
+          msg.add  " align(get, expected): " & $nim_align & " != " & $c_align
+        echo msg
 
 
+macro testOffsetOf(a,b1,b2: untyped): untyped =
+  let typeName = newLit(a.repr)
+  let member   = newLit(b2.repr)
+  result = quote do:
+    let
+      c_offset   = c_offsetof(`a`,`b1`)
+      nim_offset = offsetof(`a`,`b2`)
+    if c_offset != nim_offset:
+      echo `typeName`, ".", `member`, " offset: ", c_offset, " != ", nim_offset
 
+template testOffsetOf(a,b: untyped): untyped =
+  testOffsetOf(a,b,b)
+
+proc strAlign(arg: string): string =
+  const minLen = 22
+  result = arg
+  for i in 0 ..< minLen - arg.len:
+    result &= ' '
+
+macro c_offsetof(a: typed, b: untyped): int32 =
+  ## Buffet proof implementation that works on actual offsetof operator
+  ## in the c backend. Assuming of course this implementation is
+  ## correct.
+  let bliteral =
+    if b.kind == nnkStrLit:
+      b
+    else:
+      newLit(repr(b))
+  result = quote do:
+    var res: int32
+    {.emit: [res, " = offsetof(", `a`, ", ", `bliteral`, ");"] .}
+    res
+
+macro c_sizeof(a: typed): int32 =
+  ## Buffet proof implementation that works using the sizeof operator
+  ## in the c backend. Assuming of course this implementation is
+  ## correct.
+  result = quote do:
+    var res: int32
+    {.emit: [res, " = sizeof(", `a`, ");"] .}
+    res
+
+macro c_alignof(arg: untyped): untyped =
+  ## Buffet proof implementation that works on actual alignment
+  ## behavior measured at runtime.
+  let typeSym = genSym(nskType, "AlignTestType"&arg.repr)
+  result = quote do:
+    type
+      `typeSym` = object
+        causeAlign: byte
+        member: `arg`
+    c_offsetof(`typeSym`, member)
+
+macro testAlign(arg:untyped):untyped =
+  let prefix = newLit(arg.lineinfo & "  alignof " & arg.repr & " ")
+  result = quote do:
+    let cAlign = c_alignof(`arg`)
+    let nimAlign = alignof(`arg`)
+    if cAlign != nimAlign:
+      echo `prefix`, cAlign, " != ", nimAlign
+
+testAlign(pointer)
+testAlign(int)
+testAlign(uint)
+testAlign(int8)
+testAlign(int16)
+testAlign(int32)
+testAlign(int64)
+testAlign(uint8)
+testAlign(uint16)
+testAlign(uint32)
+testAlign(uint64)
+testAlign(float)
+testAlign(float32)
+testAlign(float64)
+
+type
+  MyEnum {.pure.} = enum
+    ValueA
+    ValueB
+    ValueC
+
+  OtherEnum {.pure, size: 8.} = enum
+    ValueA
+    ValueB
+
+  Enum1 {.pure, size: 1.} = enum
+    ValueA
+    ValueB
+
+  Enum2 {.pure, size: 2.} = enum
+    ValueA
+    ValueB
+
+  Enum4 {.pure, size: 4.} = enum
+    ValueA
+    ValueB
+
+  Enum8 {.pure, size: 8.} = enum
+    ValueA
+    ValueB
+
+testAlign(MyEnum)
+testAlign(OtherEnum)
+testAlign(Enum1)
+testAlign(Enum2)
+testAlign(Enum4)
+testAlign(Enum8)
+
+
+template testinstance(body: untyped): untyped =
+  block:
+    {.pragma: objectconfig.}
+    body
+
+  block:
+    {.pragma: objectconfig, packed.}
+    body
+
+testinstance:
+  type
+
+    EnumObjectA  {.objectconfig.} = object
+      a : Enum1
+      b : Enum2
+      c : Enum4
+      d : Enum8
+
+    EnumObjectB  {.objectconfig.} = object
+      a : Enum8
+      b : Enum4
+      c : Enum2
+      d : Enum1
+
+    TrivialType  {.objectconfig.} = object
+      x,y,z: int8
+
+    SimpleAlignment {.objectconfig.} = object
+      # behaves differently on 32bit Windows and 32bit Linux
+      a,b: int8
+      c: int64
+
+    AlignAtEnd {.objectconfig.} = object
+      a: int64
+      b,c: int8
+
+    SimpleBranch {.objectconfig.} = object
+      case kind: MyEnum
+      of MyEnum.ValueA:
+        a: int16
+      of MyEnum.ValueB:
+        b: int32
+      of MyEnum.ValueC:
+        c: int64
+
+    PaddingBeforeBranchA {.objectconfig.} = object
+      cause: int8
+      case kind: MyEnum
+      of MyEnum.ValueA:
+        a: int16
+      of MyEnum.ValueB:
+        b: int32
+      of MyEnum.ValueC:
+        c: int64
+
+    PaddingBeforeBranchB {.objectconfig.} = object
+      cause: int8
+      case kind: MyEnum
+      of MyEnum.ValueA:
+        a: int8
+      of MyEnum.ValueB:
+        b: int16
+      of MyEnum.ValueC:
+        c: int32
+
+    PaddingAfterBranch {.objectconfig.} = object
+      case kind: MyEnum
+      of MyEnum.ValueA:
+        a: int8
+      of MyEnum.ValueB:
+        b: int16
+      of MyEnum.ValueC:
+        c: int32
+      cause: int64
+
+    RecursiveStuff {.objectconfig.} = object
+      case kind: MyEnum    # packedOffset:    0
+      of MyEnum.ValueA:    # packedOffset:
+        a: int16           # packedOffset:    1
+      of MyEnum.ValueB:    # packedOffset:
+        b: int32           # packedOffset:    1
+      of MyEnum.ValueC:    # packedOffset:
+        case kind2: MyEnum # packedOffset:    1
+        of MyEnum.ValueA:  # packedOffset:
+          ca1: int8
+          ca2: int32
+        of MyEnum.ValueB:  # packedOffset:
+          cb: int32        # packedOffset:    2
+        of MyEnum.ValueC:  # packedOffset:
+          cc: int64        # packedOffset:    2
+        d1: int8
+        d2: int64
+
+    Foobar {.objectconfig.} = object
+      case kind: OtherEnum
+      of OtherEnum.ValueA:
+        a: uint8
+      of OtherEnum.ValueB:
+        b: int8
+      c: int8
+
+    Bazing {.objectconfig.} = object of RootObj
+      a: int64
+      # TODO test on 32 bit system
+      # only there the object header is smaller than the first member
+
+    InheritanceA {.objectconfig.} = object of RootObj
+      a: char
+
+    InheritanceB {.objectconfig.} = object of InheritanceA
+      b: char
+
+    InheritanceC {.objectconfig.} = object of InheritanceB
+      c: char
+
+    #Float128Test = object
+    #  a: byte
+    #  b: float128
+
+    #Bazang = object of RootObj
+    #  a: float128
+
+  const trivialSize = sizeof(TrivialType) # needs to be able to evaluate at compile time
+
+  testAlign(SimpleAlignment)
+
+  proc main(): void =
+    var t : TrivialType
+    var a : SimpleAlignment
+    var b : AlignAtEnd
+    var c : SimpleBranch
+    var d : PaddingBeforeBranchA
+    var e : PaddingBeforeBranchB
+    var f : PaddingAfterBranch
+    var g : RecursiveStuff
+    var ro : RootObj
+    var
+      e1: Enum1
+      e2: Enum2
+      e4: Enum4
+      e8: Enum8
+    var
+      eoa: EnumObjectA
+      eob: EnumObjectB
+
+    testSizeAlignOf(t,a,b,c,d,e,f,g,ro, e1, e2, e4, e8, eoa, eob)
+
+    testOffsetOf(TrivialType, x)
+    testOffsetOf(TrivialType, y)
+    testOffsetOf(TrivialType, z)
+
+    testOffsetOf(SimpleAlignment, a)
+    testOffsetOf(SimpleAlignment, b)
+    testOffsetOf(SimpleAlignment, c)
+    testOffsetOf(AlignAtEnd, a)
+    testOffsetOf(AlignAtEnd, b)
+    testOffsetOf(AlignAtEnd, c)
+
+    testOffsetOf(SimpleBranch, "_Ukind", a)
+    testOffsetOf(SimpleBranch, "_Ukind", b)
+    testOffsetOf(SimpleBranch, "_Ukind", c)
+
+    testOffsetOf(PaddingBeforeBranchA, cause)
+    testOffsetOf(PaddingBeforeBranchA, "_Ukind", a)
+    testOffsetOf(PaddingBeforeBranchB, cause)
+    testOffsetOf(PaddingBeforeBranchB, "_Ukind", a)
+
+    testOffsetOf(PaddingAfterBranch, "_Ukind", a)
+    testOffsetOf(PaddingAfterBranch, cause)
+
+    testOffsetOf(Foobar, c)
+
+    testOffsetOf(Bazing, a)
+
+    testOffsetOf(InheritanceA, a)
+    testOffsetOf(InheritanceB, b)
+    testOffsetOf(InheritanceC, c)
+
+    testOffsetOf(EnumObjectA, a)
+    testOffsetOf(EnumObjectA, b)
+    testOffsetOf(EnumObjectA, c)
+    testOffsetOf(EnumObjectA, d)
+    testOffsetOf(EnumObjectB, a)
+    testOffsetOf(EnumObjectB, b)
+    testOffsetOf(EnumObjectB, c)
+    testOffsetOf(EnumObjectB, d)
+
+    testOffsetOf(RecursiveStuff, kind)
+    testOffsetOf(RecursiveStuff, "_Ukind.S1.a",              a)
+    testOffsetOf(RecursiveStuff, "_Ukind.S2.b",              b)
+    testOffsetOf(RecursiveStuff, "_Ukind.S3.kind2",          kind2)
+    testOffsetOf(RecursiveStuff, "_Ukind.S3._Ukind2.S1.ca1", ca1)
+    testOffsetOf(RecursiveStuff, "_Ukind.S3._Ukind2.S1.ca2", ca2)
+    testOffsetOf(RecursiveStuff, "_Ukind.S3._Ukind2.S2.cb",  cb)
+    testOffsetOf(RecursiveStuff, "_Ukind.S3._Ukind2.S3.cc",  cc)
+    testOffsetOf(RecursiveStuff, "_Ukind.S3.d1",             d1)
+    testOffsetOf(RecursiveStuff, "_Ukind.S3.d2",             d2)
+
+  main()
+
+
+echo "OK"

tests/misc/tsizeof2.nim

@@ -0,0 +1,11 @@
+discard """
+errormsg: "cannot evaluate 'sizeof/alignof' because its type is not defined completely"
+line: 9
+"""
+
+type
+  MyStruct {.importc: "MyStruct".} = object
+
+const i = sizeof(MyStruct)
+
+echo i

tests/types/tillegalrecursion3.nim

@@ -0,0 +1,12 @@
+discard """
+  errormsg: "illegal recursion in type 'Foo'"
+"""
+
+type
+  Imported {.importc.} = object
+
+  Foo = object
+    b: Imported
+    a: Foo
+
+var myFoo: Foo