Merge pull request #73 from Fantu/VAL-6.0-improve

[VAL-6.0] Improvements and fixes
veeam · Nov 1, 2023 · ad1afe7 · ad1afe7
2 parents 706722f + 1e9db9d
commit ad1afe7
Show file tree

Hide file tree

Showing 7 changed files with 261 additions and 21 deletions.
diff --git a/.circleci/config.yml b/.circleci/config.yml
@@ -0,0 +1,90 @@
+# Use the latest 2.1 version of CircleCI pipeline process engine.
+# See: https://circleci.com/docs/2.0/configuration-reference
+version: 2.1
+
+shared: &shared
+    steps:
+      - checkout
+      - run:
+            name: Prepare environment
+            command: |
+                echo 'debconf debconf/frontend select Noninteractive' | debconf-set-selections
+                apt-get update
+                apt-get dist-upgrade -y
+      - run:
+            name: Install dependencies
+            command: |
+                apt-get install -y g++ cmake uuid-dev libboost-program-options-dev libboost-filesystem-dev libssl-dev debhelper
+                apt-get install -y dh-dkms || apt-get install -y dkms
+      - run:
+            name: Build blksnap-dkms package
+            working_directory: pkg/deb/blksnap-dkms
+            command: ./build.sh
+      - run:
+            name: Build blksnap-tools package
+            working_directory: pkg/deb/blksnap-tools
+            command: ./build.sh
+      - run:
+            name: Build blksnap-dev package
+            working_directory: pkg/deb/blksnap-dev
+            command: ./build.sh
+      - run:
+            name: Build blksnap-tests package
+            working_directory: pkg/deb/blksnap-tests
+            command: ./build.sh
+      - run:
+            name: Save generate packages as artifacts
+            working_directory: build
+            command: |
+                mkdir /tmp/artifacts
+                mv *.deb /tmp/artifacts
+      - store_artifacts:
+          path: /tmp/artifacts
+
+jobs:
+    debian10:
+        <<: *shared
+        docker:
+            - image: library/debian:buster
+    debian11:
+        <<: *shared
+        docker:
+            - image: library/debian:bullseye
+    debian12:
+        <<: *shared
+        docker:
+            - image: library/debian:bookworm
+    ubuntu1404:
+        <<: *shared
+        docker:
+            - image: library/ubuntu:trusty
+    ubuntu1604:
+        <<: *shared
+        docker:
+            - image: library/ubuntu:xenial
+    ubuntu1804:
+        <<: *shared
+        docker:
+            - image: library/ubuntu:bionic
+    ubuntu2004:
+        <<: *shared
+        docker:
+            - image: library/ubuntu:focal
+    ubuntu2204:
+        <<: *shared
+        docker:
+            - image: library/ubuntu:jammy
+    ubuntu2310:
+        <<: *shared
+        docker:
+            - image: library/ubuntu:mantic
+
+workflows:
+  build-deb:
+    jobs:
+      - debian10
+      - debian11
+      - debian12
+      - ubuntu2004
+      - ubuntu2204
+      - ubuntu2310
diff --git a/.github/workflows/Kernel-module.yml b/.github/workflows/Kernel-module.yml
@@ -0,0 +1,50 @@
+name: Build module
+
+on:
+  workflow_dispatch:
+  push:
+    branches:
+      - "*"
+    paths:
+      - "module/**"
+  pull_request:
+    branches:
+      - "*"
+    paths:
+      - "module/**"
+
+jobs:
+  build-on-image:
+    name: Kernel ${{ matrix.kversion }}
+    runs-on: ubuntu-latest
+    container:
+      image: ${{ matrix.image }}
+      # Disable seccomp until a container manager in GitHub recognizes
+      # clone3() syscall,
+      # <https://github.com/actions/virtual-environments/issues/3812>.
+      options: --security-opt seccomp=unconfined
+
+    strategy:
+      fail-fast: false
+      matrix:
+        include:
+          - kversion: "5.10"
+            image: "library/debian:bullseye"
+          - kversion: "6.1"
+            image: "library/debian:bookworm"
+          - kversion: "last stable"
+            image: "library/debian:sid"
+
+    steps:
+      - uses: actions/checkout@v4
+
+      - name: Install packages required
+        run: |
+         apt-get update
+         apt-get install -y linux-headers-amd64 make g++
+
+      - name: Build kernel module
+        run: |
+          cd module
+          make -Wall -C /lib/modules/*/build M=`pwd`
+
diff --git a/.github/workflows/Linux.yml b/.github/workflows/Linux.yml
@@ -0,0 +1,63 @@
+name: Test build
+
+on:
+  push:
+    branches:
+      - "*"
+    paths-ignore:
+      - "README.md"
+      - "doc/**"
+      - "patches/**"
+  pull_request:
+    branches:
+      - "*"
+    paths-ignore:
+      - "README.md"
+      - "doc/**"
+      - "patches/**"
+  workflow_dispatch:
+
+jobs:
+  build:
+    name: amd64
+    runs-on: ubuntu-22.04
+
+    strategy:
+      fail-fast: false
+
+    steps:
+      - uses: actions/checkout@v4
+
+      - name: Install packages required
+        run: |
+         sudo apt-get update
+         sudo apt-get install -y linux-headers-$(uname -r) g++ cmake uuid-dev libboost-program-options-dev libboost-filesystem-dev libssl-dev
+
+      - name: Build kernel module
+        run: |
+          cd module
+          ./mk.sh build
+
+      - name: Build blksnap-tools
+        working-directory: tools/blksnap
+        run: |
+          mkdir bin
+          cd bin
+          cmake ..
+          make
+
+      - name: Build blksnap-dev
+        working-directory: lib/blksnap
+        run: |
+          mkdir bin
+          cd bin
+          cmake ..
+          make
+
+      - name: Build blksnap-tests
+        working-directory: tests/cpp
+        run: |
+          mkdir bin
+          cd bin
+          cmake ..
+          make
diff --git a/README.md b/README.md
@@ -1,55 +1,83 @@
-# blksnap
-
-* doc/ - documentation
-* include/ - libraries public headers
-* lib/ - libraries sources
-* module/ - blksnap kernel module sources
-* patches/ - patches for thelinux kernel
-* pkg/ - scripts for building packages
-* tests/ - Test scripts and test source codes.
-* tools/ - source files of tools for working with blksnap
-
-## kernel module blksnap
+# BLKSNAP - Block Devices Snapshots Module
+
+* [Extended description and features](doc/blksnap.md)
+* [Repository structure](#repository-structure)
+* [Licensing](#licensing)
+* [Kernel module](#kernel-module)
+* [Upstream kernel integration](https://github.com/veeam/blksnap/blob/master/doc/README-upstream-kernel.md)
+* [Tools](#tools)
+* [Library](#library)
+* [Tests](#tests)
+  - Details:
+    * [Boundary](doc/tests/boundary.md)
+    * [Corrupt](doc/tests/corrupt.md)
+    * [Diff storage](doc/tests/diff_storage.md)
+* [Compatibility notes](#compatibility-notes)
+
+## Repository structure
+
+* doc/ - Documentation
+* include/ - Libraries public headers
+* lib/ - Libraries sources
+* module/ - Sources of kernel module
+* patches/ - Patches for the upstream linux kernel
+* pkg/ - Scripts for building deb and rpm packages
+* tests/ - Test scripts and tests source code
+* tools/ - Source files of tools for working with blksnap
+
+## Licensing
+
+Kernel module is GPL-2 only, tools and tests are GPL-2+, library and include are LGPL-3+.
+
+Copyright (C) 2022 Veeam Software Group GmbH
+
+This project use [SPDX License Identifier](https://spdx.dev/ids/) in source files header.
+
+## Kernel module
 This kernel module implements snapshot and changed block tracking functionality.
 The module is developed with the condition of simply adding it to the upstream.
 Therefore, the module is divided into two parts: bdevfilter and blksnap.
 bdevfilter provides the ability to intercept I/O units (bio). The main logic
 is concentrated in blksnap. The upstream variant does not contain a bdevfilter,
 but accesses the kernel to attach and detach the block device filter.
 
+Relating the work in progress for integration in upstream kernel see the specific [README](https://github.com/veeam/blksnap/blob/master/doc/README-upstream-kernel.md)
+
 ### How to build
 Installing the necessary deb packages.
 ``` bash
-sudo apt install gcc linux-headers
+sudo apt install gcc linux-headers-$(uname -r)
 ```
 Or installing the necessary rpm packages.
 ``` bash
 sudo yum install gcc kernel-devel
 ```
 ``` bash
-cd ./mobule
+cd ./module
 mk.sh build
 ```
 In directory current directory you can found bdevfilter.ko and blksnap.ko.
 
 ### How to install
 ``` bash
-cd ./mobule
+cd ./module
 mk.sh install-flt
 mk.sh install
 ```
 ### How to create deb package
 ``` bash
-sudo apt install debhelper
+sudo apt install debhelper dkms
+# on debian >=12 and ubuntu >= 23.04 is needed dh-dkms, not installed anymore as dkms dep.
+sudo apt install dh-dkms
 cd ./pkg/deb/blksnap-dkms
 build.sh ${VERSION}
 ```
 ### How to create rpm package
 There are several variants, look in the ./pkg/rpm directory.
 
-## blksnap tool
-The blksnap tool allows to manage the module from the command line.
-The program allows to execute individual ioctls of the blksnap module.
+## Tools
+The blksnap tools allows you to manage the module from the command line.
+The program allows for execution of individual ioctls of the blksnap module.
 The interface of the program may seem inconvenient to the user,
 since it is assumed that it will be called by other applications.
 ### How to build
@@ -77,7 +105,7 @@ cd ./pkg/deb/blksnap-tools
 build.sh ${VERSION}
 ```
 
-## library
+## Library
 The dynamic C library is not needed to work with blksnap. File
 ./include/blksnap/blk_snap.h contains everything you need to work with blksnap.
 But to demonstrate how to call the ioctl, a static c++ library was created.
@@ -107,7 +135,7 @@ cd ./pkg/deb/blksnap-dev
 build.sh ${VERSION}
 ```
 
-## tests
+## Tests
 The test scripts are written in bash and use the blksnap tool to control
 the blksnap module. The test scripts allow to check the main functions of
 the module. To implement complex test algorithms, С++ tests are implemented.
@@ -136,3 +164,9 @@ sudo apt install debhelper
 cd ./pkg/deb/blksnap-tests
 build.sh ${VERSION}
 ```
+
+## Compatibility notes
+- blksnap kernel module support kernel versions from 5.10 up to 6.5
+- all supported debian and ubuntu supported versions are supported but with some notes:
+  - not all have debian/ubuntu versions have official packages of kernel >= 5.10, so an unofficial or custom ones more updated are needed, with blksnap-dkms should be still possible easy/fast build/install blksnap module on them (is also possible build/install it manually without dkms)
+  - debian 8 and ubuntu 14.04 needs to install cmake 3 from backports to build
diff --git a/include/blksnap/Blksnap.h b/include/blksnap/Blksnap.h
@@ -23,6 +23,7 @@
  * flexibility. Uses structures that are directly passed to the kernel module.
  */
 
+#include <stdint.h>
 #include <string>
 #include <uuid/uuid.h>
 #include <vector>

diff --git a/include/blksnap/Service.h b/include/blksnap/Service.h
@@ -21,6 +21,7 @@
  * The hi-level abstraction for the blksnap kernel module.
  * Allows to show module kernel version.
  */
+#include <stdint.h>
 #include <string>
 
 namespace blksnap

diff --git a/lib/blksnap/Blksnap.cpp b/lib/blksnap/Blksnap.cpp
@@ -19,6 +19,7 @@
 #include <blksnap/Blksnap.h>
 #include <errno.h>
 #include <fcntl.h>
+#include <stdint.h>
 #include <stdio.h>
 #include <stdlib.h>
 #include <sys/ioctl.h>