Merge pull request #2 from wserr/feature/change-setup

Feature/change setup

.gitignore

@@ -1,8 +1,6 @@
 # Environment files for docker composes
 .env.production
+production.env
 
 # Home assistant configuration file
-configuration.production.yaml
+configuration-production.yaml
-
-.env
-run-docker.sh

configurations/eclipse-mosquitto/mosquitto.conf

@@ -1,9 +0,0 @@
-listener 1883
-persistence false
-# persistence_location /mosquitto/data/
-log_dest file /mosquitto/log/mosquitto.log
-
-## Authentication ##
-# By default, Mosquitto >=2.0 allows only authenticated connections. Change to true to enable anonymous connections.
-allow_anonymous true
-# password_file /mosquitto/config/password.txt

configurations/otel/collector-config.yaml

@@ -0,0 +1,28 @@
+receivers:
+  otlp:
+    protocols:
+      grpc:
+        endpoint: 0.0.0.0:4317
+      http:
+        endpoint: 0.0.0.0:4318
+
+processors:
+  batch:
+
+exporters:
+  prometheus:
+    endpoint: 0.0.0.0:8889
+  debug:
+    verbosity: detailed
+
+service:
+  extensions: []
+  pipelines:
+    #traces:
+      #receivers: [otlp]
+      #processors: [batch]
+      #exporters: [debug]
+    metrics:
+      receivers: [otlp]
+      processors: [batch]
+      exporters: [prometheus]

configurations/prometheus/prometheus.yml

@@ -8,8 +8,11 @@ scrape_configs:
     - targets: ["prometheus:9090"]
 
   - job_name: "node"
+    scrape_interval: 1m
     static_configs:
     - targets: ["node-exporter:9100"]
-  - job_name: "pi_node"
+
+  - job_name: "otel"
+    scrape_interval: 5s
     static_configs:
-    - targets: ["10.55.8.1:9100"]
+    - targets: ["otel-collector:8889"]

configurations/traefik/traefik.1.7.10.toml

@@ -1,55 +0,0 @@
-defaultEntryPoints = ["http"]
-
-[entryPoints]
-    [entryPoints.whoami]
-    address = ":8088"
-    [entryPoints.homeassistant]
-    address = ":8089"
-    [entryPoints.syncthing]
-    address = ":8090"
-    [entryPoints.grafana]
-    address = ":8091"
-    [entryPoints.influxdb]
-    address = ":8092"
-
-[file]
-
-[backends]
-    [backends.whoami]
-        [backends.whoami.servers.server1]
-        url = "http://localhost:2001"
-    [backends.homeassistant]
-        [backends.homeassistant.servers.server1]
-        url = "http://10.55.8.3:8123/"
-    [backends.syncthing]
-        [backends.syncthing.servers.server1]
-        url = "http://10.55.8.3:8384/"
-    [backends.grafana]
-        [backends.grafana.servers.server1]
-        url = "http://10.55.8.3:3000/"
-    [backends.influxdb]
-        [backends.influxdb.servers.server1]
-        url = "http://10.55.8.3:8086/"
-
-[frontends]
-    [frontends.whoami]
-    entryPoints = ["whoami"]
-    backend = "whoami"
-
-    [frontends.homeassistant]
-    entryPoints = ["homeassistant"]
-    backend = "homeassistant"
-
-    [frontends.syncthing]
-    entryPoints = ["syncthing"]
-    backend = "syncthing"
-
-    [frontends.grafana]
-    entryPoints = ["grafana"]
-    backend = "grafana"
-    passHostHeader = true
-    
-    [frontends.influxdb]
-    entryPoints = ["influxdb"]
-    backend = "influxdb"
-    passHostHeader = true

configurations/traefik/traefik.yml

@@ -0,0 +1,25 @@
+## traefik.yml
+
+# Docker configuration backend
+providers:
+  docker:
+    defaultRule: "Host(`{{ trimPrefix `/` .Name }}.docker.localhost`)"
+
+# API and dashboard configuration
+api:
+  insecure: true
+
+tracing:
+  capturedRequestHeaders:
+    - X-Forwarded-Prefix
+  otlp:
+    grpc:
+      endpoint: otel-collector:4317
+      insecure: true
+
+metrics:
+  otlp:
+    addEntryPointsLabels: true
+    grpc:
+      endpoint: otel-collector:4317
+      insecure: true

home-server/.env

@@ -1,19 +1,4 @@
-# General settings
+REPO_DIR=/home/pi/repos/homeAutomation
-INTERNAL_IP: 0.0.0.0
 
-# Maria DB settings
+# Make sure docker user has read/write access on this folder
-# For testing purposes only. Please create a .env.mariadb to overwrite  these variables
+STORAGE_DIR=/tmp
-MARIADB_ROOT_PASSWORD=testing
-
-MARIADB_DATABASE=homeassistant
-MARIADB_USER=homeassistant_user
-MARIADB_PASSWORD=homeassistant_pw
-
-GF_SECURITY_ADMIN_USER=test
-GF_SECURITY_ADMIN_PASSWORD=test
-GF_USERS_ALLOW_SIGN_UP=false
-
-DOCKER_INFLUXDB_INIT_USERNAME=homeassistant
-DOCKER_INFLUXDB_INIT_PASSWORD=homeassistant
-DOCKER_INFLUXDB_INIT_ORG=homeassistant
-DOCKER_INFLUXDB_INIT_BUCKET=homeassistant

home-server/docker-compose.yml

@@ -1,40 +1,63 @@
-version: "3.9"
 services:
+  traefik:
+    restart: unless-stopped
+    image: traefik:v3.2
+    labels:
+      - traefik.http.routers.dashboard.rule=Host(`traefik.pladijs`)
+      - traefik.http.services.dashboard.loadbalancer.server.port=8080
+    volumes:
+      - ${REPO_DIR}/configurations/traefik/traefik.yml:/etc/traefik/traefik.yml
+      - ${STORAGE_DIR}/certificates:/var/lib/certificates
+      - /var/run/docker.sock:/var/run/docker.sock
+    ports:
+      - 80:80
+      - 443:443
+  whoami:
+    restart: unless-stopped
+    image: traefik/whoami
+    labels:
+      - traefik.http.routers.whoami.rule=Host(`whoami.pladijs`)
+      - traefik.http.services.whoami.loadbalancer.server.port=2001
+    command:
+      - --port=2001
   db:
     image: mariadb:latest
     restart: unless-stopped
     ports:
-      - ${INTERNAL_IP}:3306:3306
+      - 3306:3306
     env_file:
-      - .env
+      - environment-variables/local.env
-      - .env.production
+      - environment-variables/production.env
     volumes:
-      - /home/willem/mariadb:/var/lib/mysql
+      - ${STORAGE_DIR}/mariadb:/var/lib/mysql
   homeassistant:
     image: "ghcr.io/home-assistant/home-assistant:stable"
-    ports:
+    labels:
-      - ${INTERNAL_IP}:8123:8123
+      - traefik.http.routers.homeassistant.rule=Host(`homeassistant.pladijs`)
+      - traefik.http.services.homeassistant.loadbalancer.server.port=8123
     volumes:
-      - /home/willem/homeassistant:/config
+      - ${STORAGE_DIR}/homeassistant:/config
-      - /home/willem/repos/homeAutomation/configurations/home-assistant/configuration.production.yaml:/config/configuration.yaml
+      - ${REPO_DIR}/configurations/home-assistant/configuration-production.yaml:/config/configuration.yaml
     restart: unless-stopped
     env_file:
-      - .env
+      - environment-variables/local.env
-      - .env.production
+      - environment-variables/production.env
   syncthing:
     image: lscr.io/linuxserver/syncthing:latest
     container_name: syncthing
+    labels:
+      - traefik.http.routers.syncthing.rule=Host(`syncthing.pladijs`)
+      - traefik.http.services.syncthing.loadbalancer.server.port=8384
     environment:
       - PUID=1000
       - PGID=1000
       - TZ=Europe/London
     volumes:
-      - /home/willem/syncthing:/data1
+      - ${STORAGE_DIR}/syncthing:/data1
     ports:
-      - ${INTERNAL_IP}:8384:8384
+      - 22000:22000/tcp
-      - ${INTERNAL_IP}:22000:22000/tcp
+      - 22000:22000/udp
-      - ${INTERNAL_IP}:22000:22000/udp
+      - 21027:21027/udp
-      - ${INTERNAL_IP}:21027:21027/udp
     restart: unless-stopped
   node-exporter:
     image: prom/node-exporter:latest
@@ -49,47 +72,61 @@ services:
       - '--path.rootfs=/rootfs'
       - '--path.sysfs=/host/sys'
       - '--collector.filesystem.mount-points-exclude=^/(sys|proc|dev|host|etc)($$|/)'
-    ports:
+    labels:
-      - ${INTERNAL_IP}:9100:9100
+      - traefik.http.routers.node-exporter.rule=Host(`node-exporter.pladijs`)
+      - traefik.http.services.node-exporter.loadbalancer.server.port=9100
   prometheus:
     image: prom/prometheus:latest
     container_name: prometheus
     restart: unless-stopped
+    labels:
+      - traefik.http.routers.prometheus.rule=Host(`prometheus.pladijs`)
+      - traefik.http.services.prometheus.loadbalancer.server.port=9090
     volumes:
-      - /home/willem/repos/homeAutomation/configurations/prometheus/prometheus.yml:/etc/prometheus/prometheus.yml
+      - ${REPO_DIR}/configurations/prometheus/prometheus.yml:/etc/prometheus/prometheus.yml
-      - /home/willem/prometheus:/prometheus
+      - ${STORAGE_DIR}/prometheus:/prometheus
     command:
       - '--config.file=/etc/prometheus/prometheus.yml'
       - '--storage.tsdb.path=/prometheus'
       - '--web.console.libraries=/etc/prometheus/console_libraries'
       - '--web.console.templates=/etc/prometheus/consoles'
       - '--web.enable-lifecycle'
-    ports:
-      - ${INTERNAL_IP}:9090:9090
   grafana:
     image: grafana/grafana:latest
-    ports:
-      - ${INTERNAL_IP}:3000:3000
     volumes:
-      - /home/willem/grafana:/var/lib/grafana
+      - ${STORAGE_DIR}/grafana:/var/lib/grafana
     env_file:
-      - .env
+      - environment-variables/local.env
-      - .env.production
+      - environment-variables/production.env
+    labels:
+      - traefik.http.routers.grafana.rule=Host(`grafana.pladijs`)
+      - traefik.http.services.grafana.loadbalancer.server.port=3000
   influxdb:
     container_name: influxdb
+    labels:
+      - traefik.http.routers.influxdb.rule=Host(`influxdb.pladijs`)
+      - traefik.http.services.influxdb.loadbalancer.server.port=8086
     image: influxdb
     restart: unless-stopped
-    ports:
-      - ${INTERNAL_IP}:8086:8086/tcp   # So we can access the WebUI
     environment:
       - TZ=Europe/Brussels
       - DOCKER_INFLUXDB_INIT_MODE=setup
     env_file:
-      - .env.production
+      - environment-variables/local.env
+      - environment-variables/production.env
     volumes:
-      - /home/willem/influxdb/data:/var/lib/influxdb2
+      - ${STORAGE_DIR}/influxdb/data:/var/lib/influxdb2
-      - /home/willem/influxdb/config/:/etc/influxdb2
+      - ${STORAGE_DIR}/influxdb/config/:/etc/influxdb2
     ulimits:
       nofile:
         soft: 32768
         hard: 32768
+  otel-collector:
+    container_name: otel-collector
+    image: otel/opentelemetry-collector-contrib:0.111.0
+    restart: unless-stopped
+    volumes:
+      - ${REPO_DIR}/configurations/otel/collector-config.yaml:/etc/otelcol-contrib/config.yaml
+# Only enable if external services present
+#     ports:
+#       - 4317:4317 # OTLP gRPC receiver

home-server/environment-variables/local.env

@@ -0,0 +1,8 @@
+GF_SECURITY_ADMIN_USER=test
+GF_SECURITY_ADMIN_PASSWORD=test
+GF_USERS_ALLOW_SIGN_UP=false
+
+DOCKER_INFLUXDB_INIT_USERNAME=homeassistant
+DOCKER_INFLUXDB_INIT_PASSWORD=homeassistant
+DOCKER_INFLUXDB_INIT_ORG=homeassistant
+DOCKER_INFLUXDB_INIT_BUCKET=homeassistant

home-server/install-docker.sh

@@ -0,0 +1,27 @@
+#!/usr/bin/sh
+
+# See https://docs.docker.com/engine/install/debian/
+#
+ # Add Docker's official GPG key:
+sudo apt-get update
+sudo apt-get install ca-certificates curl
+sudo install -m 0755 -d /etc/apt/keyrings
+sudo curl -fsSL https://download.docker.com/linux/debian/gpg -o /etc/apt/keyrings/docker.asc
+sudo chmod a+r /etc/apt/keyrings/docker.asc
+
+# Add the repository to Apt sources:
+echo \
+  "deb [arch=$(dpkg --print-architecture) signed-by=/etc/apt/keyrings/docker.asc] https://download.docker.com/linux/debian \
+  $(. /etc/os-release && echo "$VERSION_CODENAME") stable" | \
+  sudo tee /etc/apt/sources.list.d/docker.list > /dev/null
+sudo apt-get update
+
+sudo apt-get install -y docker-ce docker-ce-cli containerd.io docker-buildx-plugin docker-compose-plugin
+
+sudo groupadd docker
+sudo usermod -aG docker $USER
+
+newgrp docker
+
+# Verify: 
+# docker run hello-world

home-server/readme.md

@@ -35,3 +35,12 @@ docker compose rm db
 
 - Start the VPN client automatically after reboot
 
+## Generate self signed certificates
+
+https://stackoverflow.com/questions/10175812/how-to-generate-a-self-signed-ssl-certificate-using-openssl
+
+> non-interactive and 10 years expiration
+
+```bash
+openssl req -x509 -newkey rsa:4096 -keyout key.pem -out cert.pem -sha256 -days 3650 -nodes -subj "/C=homeassistant/ST=Belgium/L=Deerlijk/O=Willem/OU=Willem/CN=willem" -subj '/CN=homeassistant.com'
+```

microservices/readme.md

@@ -1,3 +0,0 @@
-# Microservices
-
-TODO

microservices/weather/.env

@@ -1,6 +0,0 @@
-weather_weather_api_base_url=https://api.openweathermap.org/data/2.5/weather
-weather_weather_api_key=4ce897f7ce77f84919fd990cc68d9d20
-weather_influx_db_base_url=http://192.168.1.44:8092
-weather_influx_db_token=R1BGVaxWrX_ySaovweRz6ZTBDvHa5omQkricJGmn3PrI44jpfU7411PSmHOCjGcNR7OcBjmtE9gA62boHHXoKg==
-weather_latitude=50.854019
-weather_longitude=3.355230

microservices/weather/.gitignore

@@ -15,7 +15,6 @@ Cargo.lock
 
 # App settings
 Settings.toml
-.env
 
 # Log files
 logs.*.txt

microservices/weather/Dockerfile

@@ -1,4 +1,4 @@
-FROM rust:1.67 as builder
+FROM rust:1.71 as builder
 WORKDIR /usr/src/weather
 COPY . .
 RUN cargo install --path .

microservices/weather/build-docker.sh

@@ -1,5 +0,0 @@
-#!/usr/bin/env bash
-
-docker build . -t weather-microservice:latest
-docker tag weather-microservice:latest registry.willemserruys.com/weather-microservice:latest
-docker push registry.willemserruys.com/weather-microservice:latest

microservices/weather/environment-variables/.env

@@ -0,0 +1,6 @@
+weather_weather_api_base_url=
+weather_weather_api_key=
+weather_influx_db_base_url=
+weather_influx_db_token=
+weather_latitude=
+weather_longitude=

microservices/weather/run-docker.sh

@@ -1,4 +0,0 @@
-#!/usr/bin/env bash
-file=/home/willem/repos/homeAutomation/microservices/weather/logs.$(date "+%F").txt
-
-docker run --env RUST_LOG=info --env-file /home/willem/repos/homeAutomation/microservices/weather/.env registry.willemserruys.com/weather-microservice:latest &>> $file

microservices/weather/scripts/build-docker.sh

@@ -0,0 +1,3 @@
+#!/usr/bin/env bash
+
+docker build ../ -t weather-service:latest

microservices/weather/scripts/install.sh

@@ -0,0 +1,21 @@
+#!/usr/bin/bash
+
+# This script  should be run as root
+# Build and install weather service
+
+./build-docker.sh
+
+ln -s $HOME/repos/homeAutomation/microservices/weather/run-docker.sh /usr/bin/weather-service
+
+cp weather.service /etc/systemd/system/
+cp weather.timer /etc/systemd/system/
+
+systemctl daemon-reload
+
+systemctl enable weather.service
+systemctl enable weather.timer
+
+systemctl start weather.service
+systemctl start weather.timer
+
+

microservices/weather/scripts/run-docker.sh

@@ -0,0 +1,3 @@
+#!/usr/bin/env bash
+# Make sure the weather-service-image exists locally (e.g. by running the build-docker script)
+docker run --env RUST_LOG=info --env-file /home/pi/repos/homeAutomation/microservices/weather/environment-variables/.env.production weather-service:latest

microservices/weather/scripts/weather.service

@@ -0,0 +1,8 @@
+[Unit]
+Description=Weather Service
+
+[Service]
+ExecStart=/usr/bin/weather-service
+
+[Install]
+WantedBy=multi-user.target

microservices/weather/scripts/weather.timer

@@ -0,0 +1,8 @@
+[Unit]
+Description=Run weather service each 2 minutes
+
+[Timer]
+OnCalendar=*:0/2
+
+[Install]
+WantedBy=timers.target

person-detection/readme.md

@@ -1 +0,0 @@
-# Person Detection Script

person-detection/script.py

@@ -1,132 +0,0 @@
-import cv2
-import imutils
-import numpy as np
-import argparse
-
-HOGCV = cv2.HOGDescriptor()
-HOGCV.setSVMDetector(cv2.HOGDescriptor_getDefaultPeopleDetector())
-
-def detect(frame):
-    bounding_box_cordinates, weights =  HOGCV.detectMultiScale(frame, winStride = (4, 4), padding = (8, 8), scale = 1.03)
-    
-    person = 1
-    for x,y,w,h in bounding_box_cordinates:
-        cv2.rectangle(frame, (x,y), (x+w,y+h), (0,255,0), 2)
-        cv2.putText(frame, f'person {person}', (x,y), cv2.FONT_HERSHEY_SIMPLEX, 0.5, (0,0,255), 1)
-        person += 1
-    
-    cv2.putText(frame, 'Status : Detecting ', (40,40), cv2.FONT_HERSHEY_DUPLEX, 0.8, (255,0,0), 2)
-    cv2.putText(frame, f'Total Persons : {person-1}', (40,70), cv2.FONT_HERSHEY_DUPLEX, 0.8, (255,0,0), 2)
-
-    if str(args["preview"]) == 'true':
-        cv2.imshow('output', frame)
-    return frame
-
-def humanDetector(args):
-    image_path = args["image"]
-    image_url_path = args["imageUrl"]
-    video_path = args['video']
-    if str(args["camera"]) == 'true' : camera = True 
-    else : camera = False
-    writer = None
-    if args['output'] is not None and image_path is None:
-        writer = cv2.VideoWriter(args['output'],cv2.VideoWriter_fourcc(*'MJPG'), 10, (600,600))
-    if camera:
-        print('[INFO] Opening Web Cam.')
-        detectByCamera(ouput_path,writer)
-    elif video_path is not None:
-        print('[INFO] Opening Video from path.')
-        detectByPathVideo(video_path, writer)
-    elif image_path is not None:
-        print('[INFO] Opening Image from path.')
-        detectByPathImage(image_path, args['output'])
-    elif image_url_path is not None:
-        print('[INFO] Opening Image from URL path.')
-        detectByPathUrlImage(image_url_path, args['output'])
-
-def detectByCamera(writer):   
-    video = cv2.VideoCapture(0)
-    print('Detecting people...')
-    while True:
-        check, frame = video.read()
-        frame = detect(frame)
-        if writer is not None:
-            writer.write(frame)
-        key = cv2.waitKey(1)
-        if key == ord('q'):
-            break
-    video.release()
-    cv2.destroyAllWindows()
-
-def detectByPathVideo(path, writer):
-    video = cv2.VideoCapture(path)
-    check, frame = video.read()
-    if check == False:
-        print('Video Not Found. Please Enter a Valid Path (Full path of Video Should be Provided).')
-        return
-    print('Detecting people...')
-    while video.isOpened():
-        #check is True if reading was successful 
-        check, frame =  video.read()
-        if check:
-            frame = imutils.resize(frame , width=min(800,frame.shape[1]))
-            frame = detect(frame)
-            
-            if writer is not None:
-                writer.write(frame)
-            
-            key = cv2.waitKey(1)
-            if key== ord('q'):
-                break
-        else:
-            break
-    video.release()
-    cv2.destroyAllWindows()
-def detectByCamera(writer):   
-    video = cv2.VideoCapture(0)
-    print('Detecting people...')
-    while True:
-        check, frame = video.read()
-        frame = detect(frame)
-        if writer is not None:
-            writer.write(frame)
-        key = cv2.waitKey(1)
-        if key == ord('q'):
-                break
-    video.release()
-    cv2.destroyAllWindows()
-
-def detectByPathImage(path, output_path):
-    image = cv2.imread(path)
-    image = imutils.resize(image, width = min(800, image.shape[1])) 
-    result_image = detect(image)
-    if output_path is not None:
-        cv2.imwrite(output_path, result_image)
-    cv2.waitKey(0)
-    cv2.destroyAllWindows()
-
-def detectByPathUrlImage(pathUrl, output_path):
-    image = imutils.url_to_image(pathUrl)
-    image = imutils.resize(image, width = min(800, image.shape[1])) 
-    result_image = detect(image)
-    if output_path is not None:
-        cv2.imwrite(output_path, result_image)
-    cv2.waitKey(0)
-    cv2.destroyAllWindows()
-
-def argsParser():
-    arg_parse = argparse.ArgumentParser()
-    arg_parse.add_argument("-v", "--video", default=None, help="path to Video File ")
-    arg_parse.add_argument("-i", "--image", default=None, help="path to Image File ")
-    arg_parse.add_argument("-iu", "--imageUrl", default=None, help="Url to Image File ")
-    arg_parse.add_argument("-c", "--camera", default=False, help="Set true if you want to use the camera.")
-    arg_parse.add_argument("-p", "--preview", default=True, help="Set true if you want to open preview")
-    arg_parse.add_argument("-o", "--output", type=str, help="path to optional output video file")
-    args = vars(arg_parse.parse_args())
-    return args
-
-if __name__ == "__main__":
-    HOGCV = cv2.HOGDescriptor()
-    HOGCV.setSVMDetector(cv2.HOGDescriptor_getDefaultPeopleDetector())
-    args = argsParser()
-    humanDetector(args)

raspberrypi/docker-compose.yml

@@ -1,38 +0,0 @@
-version: "3.9"
-services:
-  whoami:
-    restart: unless-stopped
-    network_mode: "host"
-    image: traefik/whoami
-    command:
-      - --port=2001
-  traefik:
-    restart: unless-stopped
-    network_mode: "host"
-    image: "traefik:1.7.10"
-    volumes:
-      - /home/pi/repos/homeAutomation/configurations/traefik/traefik.1.7.10.toml:/etc/traefik/traefik.toml
-  mosquitto:
-    restart: unless-stopped
-    image: eclipse-mosquitto:2
-    volumes:
-      - /home/pi/repos/homeAutomation/configurations/eclipse-mosquitto/:/mosquitto/config/:ro
-      - ./log/:/mosquitto/log/
-    ports:
-      - 2883:1883
-      - 9001:9001
-  node-exporter:
-    image: prom/node-exporter:latest
-    container_name: node-exporter
-    restart: unless-stopped
-    volumes:
-      - /proc:/host/proc:ro
-      - /sys:/host/sys:ro
-      - /:/rootfs:ro
-    command:
-      - '--path.procfs=/host/proc'
-      - '--path.rootfs=/rootfs'
-      - '--path.sysfs=/host/sys'
-      - '--collector.filesystem.mount-points-exclude=^/(sys|proc|dev|host|etc)($$|/)'
-    ports:
-      - 9100:9100