Ce qu’il faut retenir

Numpy et Pandas n’ont pas exactement les mêmes objectifs.

Dans la plupart des cas, NumPy peut être légèrement plus rapide que pandas, car NumPy est plus bas niveau et a moins de surcharge. Cependant, pandas offre des structures de données et des fonctionnalités plus avancées, ce qui peut faciliter le travail avec des ensembles de données complexes. Les performances relatives de NumPy et pandas dépendent également des opérations spécifiques effectuées sur les données, de sorte que les différences de performances peuvent varier en fonction des tâches spécifiques. Certaines fonctions n’existent qu’avec pandas, et qui n’ont pas d’équivalents NumPy sont : read_csv, read_excel, groupby, pivot_table, merge, concat, melt, crosstab, cut, qcut, get_dummies et applymap.

Résultats

Résultat : image générée : notez bien que j’ai appelé des fonctions “bas niveau” pour qu’on voie ce que NumPy a dans le ventre et des fonctions qui n’existent que dans pandas, que ré-implémentées en Python pur + NumPy.

Code source

Voici le code source que j’ai fait, qui appelle quelques fonctions connues de NumPy et de pandas.

import numpy as np
import pandas as pd
import time
import matplotlib.pyplot as plt

# Générer un grand ensemble de données
data_np = np.random.rand(30_000_000)
data_pd = pd.DataFrame({"values": data_np})

operations = (
    "sum",
    "mean",
    "filter",
    "cum_sum",
    "sort",
    "complex",
    "pivot",
    "group_by",
    "rolling",
)
time_np = []
time_pd = []


# Définir une fonction pour chronométrer et stocker les temps d'exécution
def measure_time(start_time, end_time, time_list):
    time_list.append(end_time - start_time)


# Effectuer les différentes opérations et mesurer les temps d'exécution
for operation in operations:
    # print(f"operation: {operation}")
    print(f"{operation}")
    if operation == "sum":
        start_time_np = time.time()
        result_np = np.sum(data_np)
        end_time_np = time.time()
        measure_time(start_time_np, end_time_np, time_np)

        start_time_pd = time.time()
        result_pd = data_pd["values"].sum()
        end_time_pd = time.time()
        measure_time(start_time_pd, end_time_pd, time_pd)

    elif operation == "mean":
        start_time_np = time.time()
        mean_np = np.mean(data_np)
        end_time_np = time.time()
        measure_time(start_time_np, end_time_np, time_np)

        start_time_pd = time.time()
        mean_pd = data_pd["values"].mean()
        end_time_pd = time.time()
        measure_time(start_time_pd, end_time_pd, time_pd)

    elif operation == "filter":
        start_time_np = time.time()
        filtered_np = data_np[data_np > 0.5]
        end_time_np = time.time()
        measure_time(start_time_np, end_time_np, time_np)

        start_time_pd = time.time()
        filtered_pd = data_pd[data_pd["values"] > 0.5]
        end_time_pd = time.time()
        measure_time(start_time_pd, end_time_pd, time_pd)

    elif operation == "cum_sum":
        start_time_np = time.time()
        cum_sum_np = np.cumsum(data_np)
        end_time_np = time.time()
        measure_time(start_time_np, end_time_np, time_np)

        start_time_pd = time.time()
        cum_sum_pd = data_pd["values"].cumsum()
        end_time_pd = time.time()
        measure_time(start_time_pd, end_time_pd, time_pd)

    elif operation == "sort":
        start_time_np = time.time()
        sorted_np = np.sort(data_np)
        end_time_np = time.time()
        measure_time(start_time_np, end_time_np, time_np)

        start_time_pd = time.time()
        sorted_pd = data_pd["values"].sort_values()
        end_time_pd = time.time()
        measure_time(start_time_pd, end_time_pd, time_pd)
    elif operation == "complex":
        # Générer des données structurées
        data_1 = np.random.randint(0, 1_000_000, (2_000, 2))
        data_2 = np.random.randint(0, 1_000_000, (2_000, 2))

        # Créer des DataFrames pandas
        df_1 = pd.DataFrame(data_1, columns=["id", "value_1"])
        df_2 = pd.DataFrame(data_2, columns=["id", "value_2"])

        # Créer des arrays structurés NumPy
        d_type = np.dtype([("id", int), ("value", int)])
        numpy_data_1 = np.array(
            list(map(tuple, data_1)), dtype=d_type
        )
        numpy_data_2 = np.array(
            list(map(tuple, data_2)), dtype=d_type
        )

        # Jointure avec NumPy
        def numpy_join(data1, data2):
            result = []
            for row1 in data1:
                for row2 in data2:
                    if row1["id"] == row2["id"]:
                        result.append(
                            (row1["id"], row1["value"], row2["value"])
                        )
            return np.array(
                result,
                dtype=[
                    ("id", int),
                    ("value_1", int),
                    ("value_2", int),
                ],
            )

        start_time_np = time.time()
        numpy_result = numpy_join(numpy_data_1, numpy_data_2)
        end_time_np = time.time()
        measure_time(
            start_time_np, end_time_np, time_np
        )  # Ajoutez cette ligne

        # Jointure avec pandas
        start_time_pd = time.time()
        pandas_result = df_1.merge(df_2, on="id")
        end_time_pd = time.time()

        measure_time(start_time_pd, end_time_pd, time_pd)
    elif operation == "pivot":
        # Générer des données structurées
        unique_ids = np.arange(0, 60_000)
        unique_groups = np.arange(0, 3)
        id_col = np.repeat(unique_ids, len(unique_groups))
        group_col = np.tile(unique_groups, len(unique_ids))
        value_col = np.random.randint(0, 100, len(id_col))
        data = np.column_stack((id_col, group_col, value_col))

        # Créer des DataFrames pandas
        df = pd.DataFrame(data, columns=["id", "group", "value"])

        # Créer des arrays structurés NumPy
        d_type = np.dtype(
            [("id", int), ("group", int), ("value", int)]
        )
        numpy_data = np.array(list(map(tuple, data)), dtype=d_type)

        # Pivot avec NumPy
        def numpy_pivot(_data, _id_col, _group_col, _value_col):
            _unique_ids = np.unique(_data[_id_col])
            _unique_groups = np.unique(_data[_group_col])

            pivot_table = np.zeros(
                (len(_unique_ids), len(_unique_groups))
            )


            for row in _data:
                id_index = np.where(_unique_ids == row[_id_col])[0][0]
                group_index = np.where(
                    _unique_groups == row[_group_col]
                )[0][0]
                pivot_table[id_index, group_index] = row[_value_col]

            return pivot_table

        start_time_np = time.time()
        numpy_pivot_table = numpy_pivot(
            numpy_data, "id", "group", "value"
        )
        end_time_np = time.time()
        measure_time(start_time_np, end_time_np, time_np)

        # Pivot avec pandas
        start_time_pd = time.time()
        pandas_pivot_table = df.pivot(
            index="id", columns="group", values="value"
        )
        end_time_pd = time.time()
        measure_time(start_time_pd, end_time_pd, time_pd)

    elif operation == "group_by":
        # Générer des données structurées
        data = np.random.randint(0, 10_000_000, (100_000, 2))

        # Créer des DataFrames pandas
        df = pd.DataFrame(data, columns=["id", "value"])

        # Créer des arrays structurés NumPy
        d_type = np.dtype([("id", int), ("value", int)])
        numpy_data = np.array(list(map(tuple, data)), dtype=d_type)

        # Group_by avec NumPy
        def numpy_group_by_mean(_data):
            _unique_ids, counts = np.unique(
                _data["id"], return_counts=True
            )
            sums = np.zeros_like(_unique_ids, dtype=float)
            for row in _data:
                sums[np.where(_unique_ids == row["id"])[0][0]] += row[
                    "value"
                ]
            return _unique_ids, sums / counts

        start_time_np = time.time()
        numpy_result = numpy_group_by_mean(numpy_data)
        end_time_np = time.time()
        measure_time(start_time_np, end_time_np, time_np)

        # Group_by avec pandas
        start_time_pd = time.time()
        pandas_result = df.groupby("id")["value"].mean()
        end_time_pd = time.time()
        measure_time(start_time_pd, end_time_pd, time_pd)

    elif operation == "rolling":
        # Générer un grand ensemble de données
        data_np = np.random.rand(100_000_000)
        data_pd = pd.DataFrame({"values": data_np})

        window = 100

        def numpy_rolling_mean(arr, _window):
            _cum_sum = np.cumsum(np.insert(arr, 0, 0))
            return (
                _cum_sum[_window:] - _cum_sum[:-_window]
            ) / _window

        start_time_np = time.time()
        numpy_result = numpy_rolling_mean(data_np, window)
        end_time_np = time.time()
        measure_time(start_time_np, end_time_np, time_np)

        # Rolling avec pandas
        start_time_pd = time.time()
        pandas_result = (
            data_pd["values"].rolling(window=window).mean()
        )
        end_time_pd = time.time()
        measure_time(start_time_pd, end_time_pd, time_pd)

# Créer un graphique de comparaison
x = np.arange(len(operations))
width = 0.35

fig, ax = plt.subplots()

rects1 = ax.bar(
    x - width / 2,
    time_np,
    width,
    label="NumPy",
    color="#c9daf8",
    edgecolor="black",
    hatch="//",
    linewidth=1,
)
rects2 = ax.bar(
    x + width / 2,
    time_pd,
    width,
    label="pandas",
    color="#c2e8b8",
    edgecolor="black",
    hatch=".",
    linewidth=1,
    alpha=0.5,
)


# Modification de la taille des marqueurs dans rects2
for rect in rects2:
    rect.set_linewidth(2)

ax.set_yscale("log")
ax.set_ylabel("Temps d'exécution (s) - Échelle logarithmique")
ax.set_title(
    "Comparaison des temps d'exécution entre NumPy et pandas"
)
ax.set_xticks(x)
ax.set_xticklabels(operations)
ax.legend()


def autolabel(rects):
    for _rect in rects:
        height = _rect.get_height()
        ax.annotate(
            "{:.2f}".format(height),
            xy=(_rect.get_x() + _rect.get_width() / 2, height),
            xytext=(0, 3),  # 3 points vertical offset
            textcoords="offset points",
            ha="center",
            va="bottom",
        )


autolabel(rects1)
autolabel(rects2)

fig.tight_layout()
plt.savefig("pandas_vs_numpy.png")