How to Create a Packed Data Structure in Swift

How do I create a packed data structure in Swift?

I started writing a Swift class modeled after Python's struct module, it can be found on github as MVPCStruct.

Code for the first prototype was as follows:

enum Endianness {
case littleEndian
case bigEndian
}

// Split a large integer into bytes.
extension Int {
func splitBytes(endianness: Endianness, size: Int) -> UInt8[] {
var bytes = UInt8[]()
var shift: Int
var step: Int
if endianness == .littleEndian {
shift = 0
step = 8
} else {
shift = (size - 1) * 8
step = -8
}
for count in 0..size {
bytes.append(UInt8((self >> shift) & 0xff))
shift += step
}
return bytes
}
}
extension UInt {
func splitBytes(endianness: Endianness, size: Int) -> UInt8[] {
var bytes = UInt8[]()
var shift: Int
var step: Int
if endianness == .littleEndian {
shift = 0
step = 8
} else {
shift = Int((size - 1) * 8)
step = -8
}
for count in 0..size {
bytes.append(UInt8((self >> UInt(shift)) & 0xff))
shift = shift + step
}
return bytes
}
}

class Struct: NSObject {

//class let PAD_BYTE = UInt8(0x00) // error: class variables not yet supported
//class let ERROR_PACKING = -1

class func platformEndianness() -> Endianness {
return .littleEndian
}

// Pack an array of data according to the format string. Return NSData
// or nil if there's an error.
class func pack(format: String, data: AnyObject[], error: NSErrorPointer) -> NSData? {
let PAD_BYTE = UInt8(0x00)
let ERROR_PACKING = -1

var bytes = UInt8[]()
var index = 0
var repeat = 0
var alignment = true
var endianness = Struct.platformEndianness()

// Set error message and return nil.
func failure(message: String) -> NSData? {
if error {
error.memory = NSError(domain: "se.gu.it.GUStructPacker",
code: ERROR_PACKING,
userInfo: [NSLocalizedDescriptionKey: message])
}
return nil
}

// If alignment is requested, emit pad bytes until alignment is
// satisfied.
func padAlignment(size: Int) {
if alignment {
let mask = size - 1
while (bytes.count & mask) != 0 {
bytes.append(PAD_BYTE)
}
}
}

for c in format {
// Integers are repeat counters. Consume and continue.
if let value = String(c).toInt() {
repeat = repeat * 10 + value
continue
}
// Process repeat count values, minimum of 1.
for i in 0..(repeat > 0 ? repeat : 1) {
switch c {

case "@":
endianness = Struct.platformEndianness()
alignment = true
case "=":
endianness = Struct.platformEndianness()
alignment = false
case "<":
endianness = Endianness.littleEndian
alignment = false
case ">":
endianness = Endianness.bigEndian
alignment = false
case "!":
endianness = Endianness.bigEndian
alignment = false

case "x":
bytes.append(PAD_BYTE)

default:

if index >= data.count {
return failure("expected at least \(index) items for packing, got \(data.count)")
}
let rawValue: AnyObject = data[index++]

switch c {

case "c":
if let str = rawValue as? String {
let codePoint = str.utf16[0]
if codePoint < 128 {
bytes.append(UInt8(codePoint))
} else {
return failure("char format requires String of length 1")
}
} else {
return failure("char format requires String of length 1")
}

case "b":
if let value = rawValue as? Int {
if value >= -0x80 && value <= 0x7f {
bytes.append(UInt8(value & 0xff))
} else {
return failure("value outside valid range of Int8")
}
} else {
return failure("cannot convert argument to Int")
}

case "B":
if let value = rawValue as? UInt {
if value > 0xff {
return failure("value outside valid range of UInt8")
} else {
bytes.append(UInt8(value))
}
} else {
return failure("cannot convert argument to UInt")
}

case "?":
if let value = rawValue as? Bool {
if value {
bytes.append(UInt8(1))
} else {
bytes.append(UInt8(0))
}
} else {
return failure("cannot convert argument to Bool")
}

case "h":
if let value = rawValue as? Int {
if value >= -0x8000 && value <= 0x7fff {
padAlignment(2)
bytes.extend(value.splitBytes(endianness, size: 2))
} else {
return failure("value outside valid range of Int16")
}
} else {
return failure("cannot convert argument to Int")
}

case "H":
if let value = rawValue as? UInt {
if value > 0xffff {
return failure("value outside valid range of UInt16")
} else {
padAlignment(2)
bytes.extend(value.splitBytes(endianness, size: 2))
}
} else {
return failure("cannot convert argument to UInt")
}

case "i", "l":
if let value = rawValue as? Int {
if value >= -0x80000000 && value <= 0x7fffffff {
padAlignment(4)
bytes.extend(value.splitBytes(endianness, size: 4))
} else {
return failure("value outside valid range of Int32")
}
} else {
return failure("cannot convert argument to Int")
}

case "I", "L":
if let value = rawValue as? UInt {
if value > 0xffffffff {
return failure("value outside valid range of UInt32")
} else {
padAlignment(4)
bytes.extend(value.splitBytes(endianness, size: 4))
}
} else {
return failure("cannot convert argument to UInt")
}

case "q":
if let value = rawValue as? Int {
padAlignment(8)
bytes.extend(value.splitBytes(endianness, size: 8))
} else {
return failure("cannot convert argument to Int")
}

case "Q":
if let value = rawValue as? UInt {
padAlignment(8)
bytes.extend(value.splitBytes(endianness, size: 8))
} else {
return failure("cannot convert argument to UInt")
}

case "f", "d":
assert(false, "float/double unimplemented")

case "s", "p":
assert(false, "cstring/pstring unimplemented")

case "P":
assert(false, "pointer unimplemented")

default:
return failure("bad character in format")
}
}
}
// Reset the repeat counter.
repeat = 0
}

if index != data.count {
return failure("expected \(index) items for packing, got \(data.count)")
}
return NSData(bytes: bytes, length: bytes.count)
}

}

Using Structs (Bytes) with SWIFT - Struct to NSData and NSData to Struct

Change Byte! to Byte.

Otherwise you are creating struct holding Optional<Byte> which is larger than one byte because in addition to hold a byte, it need extra byte to indicate if it is nil

Note: This may still not working as you may need something like __attribute__((packed)) to tell compiler how to deal with alignment. AFAIK, it is not available in Swift.

Metal/Metal2 + Swift: How to pass complex Swift structure as shader argument?

(Unless they've recently added something to the language definition), Swift does not guarantee the memory layout of structs in raw memory, either the byte order, element order, any padding, or even whether a struct is contiguous (and not broken up into non-adjacent sub-blocks).

So the most robust solution purely in Swift (and not just the one that works with the current version of the compilers you are testing) is to use (unsafe)raw(buffer)pointers, and manually pack bytes into the Metal buffers.

Or just use a C struct and C subroutines to pack it (all callable from Swift using a bridging header).

Does Swift guarantee the storage order of fields in classes and structs?

Yes, the order of the struct elements in memory is the order of
their declaration. The details can be found
in Type Layout
(emphasis added). Note however the use of "currently", so this
may change in a future version of Swift:

Fragile Struct and Tuple Layout

Structs and tuples currently share the same layout algorithm, noted as the "Universal" layout algorithm in the compiler implementation. The algorithm is as follows:

  • Start with a size of 0 and an alignment of 1.
  • Iterate through the
    fields, in element order for tuples, or in var declaration order for
    structs. For each field:

    • Update size by rounding up to the alignment
      of the field, that is, increasing it to the least value greater or
      equal to size and evenly divisible by the alignment of the field.
    • Assign the offset of the field to the current value of size.
    • Update
      size by adding the size of the field.
    • Update alignment to the max of
      alignment and the alignment of the field.
  • The final size and alignment
    are the size and alignment of the aggregate. The stride of the type is
    the final size rounded up to alignment.

The padding/alignment is different from C:

Note that this differs from C or LLVM's normal layout rules in that size and stride are distinct; whereas C layout requires that an embedded struct's size be padded out to its alignment and that nothing be laid out there, Swift layout allows an outer struct to lay out fields in the inner struct's tail padding, alignment permitting.

Only if a struct is imported from C then it is guaranteed to have
the same memory layout. Joe Groff from Apple writes at
[swift-users] Mapping C semantics to Swift

If you depend on a specific layout, you should define the struct in C and import it into Swift for now.

and later in that discussion:

You can leave the struct defined in C and import it into Swift. Swift will respect C's layout.

Example:

struct A {
var a: UInt8 = 0
var b: UInt32 = 0
var c: UInt8 = 0
}

struct B {
var sa: A
var d: UInt8 = 0
}

// Swift 2:
print(sizeof(A), strideof(A)) // 9, 12
print(sizeof(B), strideof(B)) // 10, 12

// Swift 3:
print(MemoryLayout<A>.size, MemoryLayout<A>.stride) // 9, 12
print(MemoryLayout<B>.size, MemoryLayout<B>.stride) // 10, 12

Here var d: UInt8 is layed out in the tail padding of var sa: A.
If you define the same structures in C

struct  CA {
uint8_t a;
uint32_t b;
uint8_t c;
};

struct CB {
struct CA ca;
uint8_t d;
};

and import it to Swift then

// Swift 2:
print(sizeof(CA), strideof(CA)) // 9, 12
print(sizeof(CB), strideof(CB)) // 13, 16

// Swift 3:
print(MemoryLayout<CA>.size, MemoryLayout<CA>.stride) // 12, 12
print(MemoryLayout<CB>.size, MemoryLayout<CB>.stride) // 16, 16

because uint8_t d is layed out after the tail padding of struct CA sa.

As of Swift 3, both size and stride return the same value
(including the struct padding) for structures imported from C,
i.e. the same value as sizeof in C would return.

Here is a simple function which helps to demonstrate the above (Swift 3):

func showMemory<T>(_ ptr: UnsafePointer<T>) {
let data = Data(bytes: UnsafeRawPointer(ptr), count: MemoryLayout<T>.size)
print(data as NSData)
}

The structures defined in Swift:

var a = A(a: 0xaa, b: 0x, c: 0xcc)
showMemory(&a) // <aa000000 cc>

var b = B(sa: a, d: 0xdd)
showMemory(&b) // <aa000000 ccdd>

The structures imported from C:

var ca = CA(a: 0xaa, b: 0x, c: 0xcc)
showMemory(&ca) // <aa000000 cc000000>

var cb = CB(ca: ca, d: 0xdd)
showMemory(&cb) // <aa000000 cc000000 dd000000>

Swift struct external padding

float2 is the Swift mapping of the C type simd_float2, which is
defined in <simd/vector_types.h as

/*! @abstract A vector of two 32-bit floating-point numbers.
* @description In C++ and Metal, this type is also available as
* simd::float2. The alignment of this type is greater than the alignment
* of float; if you need to operate on data buffers that may not be
* suitably aligned, you should access them using simd_packed_float2
* instead. */
typedef __attribute__((__ext_vector_type__(2))) float simd_float2;

The key point is

The alignment of this type is greater than the alignment of float

and you can verify that with

print(MemoryLayout<float>.alignment)  // 4
print(MemoryLayout<float2>.alignment) // 8

This causes the alignment of the Swift type Info to by 8, and its stride (i.e. the offset
in bytes between contiguous Info elements when stored in an array) to be 32.

print(MemoryLayout<Info>.alignment) // 8
print(MemoryLayout<Info>.stride) // 32

On the other hand, the C type struct Info has only float and int
members, which all have an alignment of 4 bytes. Without the
final float pad; member, the offset between contiguous elements of this type in an array is 28 bytes, not 32.

That explains the difference. What you actually should do is to define
the type in C only, and import that definition into Swift. This is
the only way which is guaranteed to preserve the memory layout,
as Joe Groff from Apple writes at
Mapping C semantics to Swift:

If you depend on a specific layout, you should define the struct in C and import it into Swift for now.

Convert Swift struct to Data bytes

I found a way to successfully push the data in correct format:

Set the values:

var cmdTypeCopy:UInt8 = BLECmdType.CMD_DATA_REQ_START
var imageInfo = BLEDataInfo(dataSizeBytes: 100000, info: 10)
var cmdLenCopy = 10
var cmdDataCopy:Array<UInt8> = [1,2,3,4,5,6,7,8,9,10]

Create Data:

var cmdTypeData = Data(bytes: &cmdTypeCopy, count: MemoryLayout<UInt8>.stride)
var imageInfoData = Data(bytes: &imageInfo, count: MemoryLayout<BLEImageInfo>.stride)
var cmdLenData = Data(bytes: &cmdLenCopy, count: MemoryLayout<UInt8>.stride)
var cmdDataData = Data(bytes: &cmdDataCopy, count: 10)

Then append one by one:

cmdTypeData.append(imageInfoData)
cmdTypeData.append(cmdLenData)
cmdTypeData.append(cmdDataData)

This worked perfectly and firmware got the data in correct format.

Difference between packed vs normal data type

This information is from here

float4 has an alignment of 16 bytes. This means that the memory address of such a type (e.g. 0x12345670) will be divisible by 16 (aka the last hexadecimal digit is 0).

packed_float4 on the other hand has an alignment of 4 bytes. Last digit of the address will be 0, 4, 8 or c

This does matter when you create custom structs. Say you want a struct with 2 normal floats and 1 float4/packed_float4:

struct A{
float x, y;
float4 z;
}

struct B{
float x, y;
packed_float4 z;
}

For A: The alignment of float4 has to be 16 and since float4 has to be after the normal floats, there is going to be 8 bytes of empty space between y and z. Here is what A looks like in memory:

 Address | 0x200 | 0x204 | 0x208 | 0x20c | 0x210 | 0x214 | 0x218 | 0x21c |
Content | x | y | - | - | z1 | z2 | z3 | z4 |
^Has to be 16 byte aligned

For B: Alignment of packed_float4 is 4, the same as float, so it can follow right after the floats in any case:

 Address | 0x200 | 0x204 | 0x208 | 0x20c | 0x210 | 0x214 |
Content | x | y | z1 | z2 | z3 | z4 |

As you can see, A takes up 32 bytes whereas B only uses 24 bytes. When you have an array of those structs, A will take up 8 more bytes for every element. So for passing around a lot of data, the latter is preferred.

The reason you need float4 at all is because the GPU can't handle 4 byte aligned packed_float4s, you won't be able to return packed_float4 in a shader. This is because of performance I assume.

One last thing: When you declare the Swift version of a struct:

struct S {
let x, y: Float
let z : (Float, Float, Float, Float)
}

This struct will be equal to B in Metal and not A. A tuple is like a packed_floatN.

All of this also applies to other vector types such as packed_float3, packed_short2, ect.

Swift 5 : Create JSON with Codable / Struct from my variables

As you have full controll over your structure and there is no collection involved i would recommend to put everything in one struct instead of scattering it over many different:

struct UserProfile: Codable{
var id: String
var name: String
var lastname: String
var street: String
var country: String
}

Regarding caching. As you can mark this struct Codable it can be easily stored in Userdefaults as Data. This extension on Userdefaults should allow you to access UserProfile in a typesafe manner.

extension UserDefaults{
var userProfile: UserProfile?{
get{
guard let data = data(forKey: "userProfile") else{
return nil
}

return try? JSONDecoder().decode(UserProfile.self, from: data)
}
set{
set(try? JSONEncoder().encode(newValue), forKey: "userProfile")
}
}
}

Usage example:

//Write
UserDefaults.standard.userProfile = UserProfile(id: UUID().uuidString, name: "First", lastname: "Last", street: "nowhere", country: "atlantis")
//Read
let profile = UserDefaults.standard.userProfile

Edit:
Editing example:

As this is a struct it gets copied every time something changes.
So read the value in a var. Modify it and save it to the defaultStore.

var profile = UserDefaults.standard.userProfile
profile?.country = "newCountry"
UserDefaults.standard.userProfile = profile


Related Topics



Leave a reply



Submit